This is an automated email from the ASF dual-hosted git repository. peacewong pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/linkis.git
commit 63249785086b53f18576a071b8522236d88167ee Merge: ba7e6f603 3cd9d3618 Author: peacewong <[email protected]> AuthorDate: Tue Jun 20 19:40:49 2023 +0800 Merge 1.4.0 to Master Merge 1.4.0 to Master .github/actions/chart-testing-action | 1 + .github/actions/kind-action | 1 + .github/workflows/build-backend.yml | 24 +- .github/workflows/check-sql-pg-script.yml | 47 + .github/workflows/integration-test.yml | 188 ++++ .github/workflows/publish-docker.yaml | 10 +- .gitmodules | 6 + README.md | 8 +- README_CN.md | 27 +- docs/configuration/impala.md | 28 + .../configuration/linkis-computation-engineconn.md | 2 +- .../linkis-computation-governance-common.md | 4 +- docs/configuration/linkis-manager-common.md | 2 +- docs/configuration/linkis-metadata-query.md | 10 + docs/configuration/linkis-udf.md | 2 +- docs/configuration/spark.md | 9 + docs/errorcode/linkis-configuration-errorcode.md | 2 +- docs/impala-usage.md | 155 +++ docs/info-1.4.0.md | 5 + docs/trino-usage.md | 4 +- .../linkis/common/exception/ErrorException.java | 2 +- .../linkis/common/exception/FatalException.java | 2 +- .../common/exception/LinkisRetryException.java | 4 +- .../java/org/apache/linkis/common/io/FsPath.java | 20 +- .../apache/linkis/common/utils/SecurityUtils.java | 7 +- .../common/utils/VariableOperationUtils.java | 1 + .../linkis/common/conf/BDPConfiguration.scala | 11 +- .../apache/linkis/common/conf/Configuration.scala | 2 + .../linkis/common/conf/BDPConfigurationTest.java | 41 + .../org/apache/linkis/common/io/FsPathTest.java | 66 ++ .../linkis/common/utils/ByteTimeUtilsTest.java | 47 + .../linkis/common/utils/SecurityUtilsTest.java | 13 - .../common/variable/VariableOperationTest.java | 8 + .../src/test/resources/linkis.properties | 16 + .../linkis/hadoop/common/conf/HadoopConf.scala | 12 + .../common/entity/HDFSFileSystemContainer.scala | 4 +- .../linkis/hadoop/common/utils/HDFSUtils.scala | 160 ++- .../linkis/hadoop/common/conf/HDFSUtilsTest.scala | 55 + linkis-commons/linkis-httpclient/pom.xml | 25 - linkis-commons/linkis-module/pom.xml | 154 --- .../org/apache/linkis/proxy/ProxyUserService.java | 23 - .../apache/linkis/server/ticket/RedisClient.scala | 4 +- .../linkis/server/utils/LinkisMainHelper.scala | 2 +- linkis-commons/linkis-mybatis/pom.xml | 28 +- .../mybatis/MybatisConfigurationFactory.java | 4 +- .../linkis/mybatis/conf/MybatisConfiguration.java | 2 + .../protocol/callback/LogCallbackProtocol.scala | 3 - linkis-commons/linkis-rpc/pom.xml | 18 +- .../rpc/transform/JavaCollectionSerializer.scala | 50 - .../apache/linkis/rpc/transform/RPCFormats.scala | 26 - .../apache/linkis/rpc/transform/RPCProduct.scala | 55 - .../org/apache/linkis/rpc/RPCFormatsTest.scala | 93 -- linkis-commons/linkis-storage/pom.xml | 34 +- .../java/org/apache/linkis/storage/FSFactory.java | 96 ++ .../org/apache/linkis/storage/LineMetaData.java | 44 + .../java/org/apache/linkis/storage/LineRecord.java | 47 + .../linkis/storage/conf/LinkisStorageConf.java | 65 ++ .../org/apache/linkis/storage/csv/CSVFsWriter.java | 35 + .../linkis/storage/csv/StorageCSVWriter.java | 128 +++ .../org/apache/linkis/storage/domain/Column.java | 51 + .../org/apache/linkis/storage/domain/DataType.java | 226 ++++ .../org/apache/linkis/storage/domain/Dolphin.java | 130 +++ .../apache/linkis/storage/domain/MethodEntity.java | 132 +++ .../storage/domain/MethodEntitySerializer.java | 84 ++ .../storage/excel/ExcelAnalysisException.java | 34 + .../apache/linkis/storage/excel/ExcelFsWriter.java | 41 + .../linkis/storage/excel/ExcelXlsReader.java | 278 +++++ .../apache/linkis/storage/excel/FirstRowDeal.java | 55 + .../apache/linkis/storage/excel/IExcelRowDeal.java | 26 + .../apache/linkis/storage/excel/RowToCsvDeal.java | 66 ++ .../linkis/storage/excel/StorageExcelWriter.java | 310 ++++++ .../storage/excel/StorageMultiExcelWriter.java | 58 + .../org/apache/linkis/storage/excel/XlsUtils.java | 1 + .../storage/factory/impl/BuildHDFSFileSystem.java | 6 +- .../storage/factory/impl/BuildLocalFileSystem.java | 8 +- .../storage/factory/impl/BuildOSSSystem.java | 70 ++ .../storage/factory/impl/BuildS3FileSystem.java | 62 ++ .../org/apache/linkis/storage/fs/FileSystem.java | 6 + .../linkis/storage/fs/impl/HDFSFileSystem.java | 42 +- .../linkis/storage/fs/impl/LocalFileSystem.java | 27 +- .../linkis/storage/fs/impl/OSSFileSystem.java | 394 +++++++ .../linkis/storage/fs/impl/S3FileSystem.java | 367 +++++++ .../org/apache/linkis/storage/io/IOClient.java | 29 + .../apache/linkis/storage/io/IOClientFactory.java | 57 + .../storage/io/IOMethodInterceptorCreator.java | 25 + .../storage/io/IOMethodInterceptorFactory.java | 49 + .../storage/resultset/DefaultResultSetFactory.java | 188 ++++ .../linkis/storage/resultset/ResultMetaData.java | 22 + .../linkis/storage/resultset/ResultRecord.java | 22 + .../linkis/storage/resultset/ResultSetFactory.java | 74 ++ .../storage/resultset/ResultSetReaderFactory.java | 115 ++ .../storage/resultset/ResultSetWriterFactory.java | 88 ++ .../linkis/storage/resultset/StorageResultSet.java | 79 ++ .../storage/resultset/StorageResultSetReader.java | 180 ++++ .../storage/resultset/StorageResultSetWriter.java | 260 +++++ .../storage/resultset/html/HtmlResultSet.java | 48 + .../linkis/storage/resultset/io/IOMetaData.java | 37 + .../linkis/storage/resultset/io/IORecord.java | 35 + .../storage/resultset/io/IOResultDeserializer.java | 37 + .../storage/resultset/io/IOResultSerializer.java | 47 + .../linkis/storage/resultset/io/IOResultSet.java | 43 + .../resultset/picture/PictureResultSet.java | 48 + .../storage/resultset/table/TableMetaData.java | 44 + .../storage/resultset/table/TableRecord.java | 44 + .../resultset/table/TableResultDeserializer.java | 100 ++ .../resultset/table/TableResultSerializer.java | 111 ++ .../storage/resultset/table/TableResultSet.java | 44 + .../resultset/txt/TextResultDeserializer.java | 36 + .../resultset/txt/TextResultSerializer.java | 50 + .../storage/resultset/txt/TextResultSet.java | 46 + .../apache/linkis/storage/script/Compaction.java | 44 + .../org/apache/linkis/storage/script/Parser.java | 30 + .../linkis/storage/script/ParserFactory.java | 31 + .../linkis/storage/script/ScriptFsReader.java | 40 + .../linkis/storage/script/ScriptFsWriter.java | 37 + .../linkis/storage/script/ScriptMetaData.java | 41 + .../apache/linkis/storage/script/ScriptRecord.java | 26 + .../org/apache/linkis/storage/script/Variable.java | 48 + .../linkis/storage/script/VariableParser.java | 131 +++ .../script/compaction/CommonScriptCompaction.java | 47 + .../script/compaction/PYScriptCompaction.java | 45 + .../script/compaction/QLScriptCompaction.java | 45 + .../script/compaction/ScalaScriptCompaction.java | 45 + .../script/compaction/ShellScriptCompaction.java | 45 + .../storage/script/parser/CommonScriptParser.java | 75 ++ .../storage/script/parser/PYScriptParser.java | 45 + .../storage/script/parser/QLScriptParser.java | 45 + .../storage/script/parser/ScalaScriptParser.java | 45 + .../storage/script/parser/ShellScriptParser.java | 45 + .../script/reader/StorageScriptFsReader.java | 161 +++ .../script/writer/StorageScriptFsWriter.java | 124 +++ .../linkis/storage/source/AbstractFileSource.java | 109 ++ .../apache/linkis/storage/source/FileSource.java | 172 +++ .../apache/linkis/storage/source/FileSplit.java | 281 +++++ .../linkis/storage/source/ResultsetFileSource.java | 54 + .../linkis/storage/source/TextFileSource.java | 71 ++ .../linkis/storage/utils/FileSystemUtils.java | 136 +++ .../linkis/storage/utils/StorageConfiguration.java | 161 +++ .../apache/linkis/storage/utils/StorageHelper.java | 156 +++ .../apache/linkis/storage/utils/StorageUtils.java | 270 +++++ .../org/apache/linkis/storage/FSFactory.scala | 98 -- .../org/apache/linkis/storage/LineMetaData.scala | 32 - .../org/apache/linkis/storage/LineRecord.scala | 34 - .../linkis/storage/conf/LinkisStorageConf.scala | 63 -- .../apache/linkis/storage/csv/CSVFsReader.scala | 22 - .../apache/linkis/storage/csv/CSVFsWriter.scala | 39 - .../linkis/storage/csv/StorageCSVReader.scala | 20 - .../linkis/storage/csv/StorageCSVWriter.scala | 93 -- .../apache/linkis/storage/domain/DataType.scala | 148 --- .../org/apache/linkis/storage/domain/Dolphin.scala | 104 -- .../linkis/storage/domain/MethodEntity.scala | 101 -- .../linkis/storage/excel/ExcelFsReader.scala | 22 - .../linkis/storage/excel/ExcelFsWriter.scala | 47 - .../linkis/storage/excel/ExcelXlsReader.java | 378 ------- .../linkis/storage/excel/StorageExcelReader.scala | 20 - .../linkis/storage/excel/StorageExcelWriter.scala | 236 ----- .../storage/excel/StorageMultiExcelWriter.scala | 53 - .../org/apache/linkis/storage/io/IOClient.scala | 75 -- .../storage/io/IOMethodInterceptorCreator.scala | 53 - .../resultset/DefaultResultSetFactory.scala | 130 --- .../linkis/storage/resultset/ResultMetaData.scala | 22 - .../linkis/storage/resultset/ResultRecord.scala | 22 - .../storage/resultset/ResultSetFactory.scala | 69 -- .../linkis/storage/resultset/ResultSetReader.scala | 91 -- .../linkis/storage/resultset/ResultSetWriter.scala | 74 -- .../storage/resultset/StorageResultSet.scala | 50 - .../storage/resultset/StorageResultSetReader.scala | 154 --- .../storage/resultset/StorageResultSetWriter.scala | 221 ---- .../storage/resultset/html/HtmlResultSet.scala | 34 - .../linkis/storage/resultset/io/IOMetaData.scala | 25 - .../linkis/storage/resultset/io/IORecord.scala | 25 - .../resultset/io/IOResultDeserializer.scala | 36 - .../storage/resultset/io/IOResultSerializer.scala | 43 - .../linkis/storage/resultset/io/IOResultSet.scala | 32 - .../resultset/picture/PictureResultSet.scala | 34 - .../storage/resultset/table/TableMetaData.scala | 30 - .../storage/resultset/table/TableRecord.scala | 36 - .../resultset/table/TableResultDeserializer.scala | 90 -- .../resultset/table/TableResultSerializer.scala | 92 -- .../storage/resultset/table/TableResultSet.scala | 32 - .../resultset/txt/TextResultDeserializer.scala | 34 - .../resultset/txt/TextResultSerializer.scala | 46 - .../storage/resultset/txt/TextResultSet.scala | 33 - .../linkis/storage/script/ScriptFsReader.scala | 37 - .../linkis/storage/script/ScriptFsWriter.scala | 115 -- .../linkis/storage/script/VariableParser.scala | 117 --- .../script/compaction/CommonScriptCompaction.scala | 34 - .../script/compaction/PYScriptCompaction.scala | 39 - .../script/compaction/QLScriptCompaction.scala | 38 - .../script/compaction/ScalaScriptCompaction.scala | 38 - .../script/compaction/ShellScriptCompaction.scala | 39 - .../storage/script/parser/CommonScriptParser.scala | 63 -- .../storage/script/parser/PYScriptParser.scala | 39 - .../storage/script/parser/QLScriptParser.scala | 38 - .../storage/script/parser/ScalaScriptParser.scala | 39 - .../storage/script/parser/ShellScriptParser.scala | 39 - .../script/reader/StorageScriptFsReader.scala | 126 --- .../script/writer/StorageScriptFsWriter.scala | 120 --- .../linkis/storage/source/AbstractFileSource.scala | 71 -- .../apache/linkis/storage/source/FileSource.scala | 154 --- .../apache/linkis/storage/source/FileSplit.scala | 162 --- .../storage/source/ResultsetFileSource.scala | 36 - .../linkis/storage/source/TextFileSource.scala | 52 - .../linkis/storage/utils/FileSystemUtils.scala | 116 -- .../storage/utils/StorageConfiguration.scala | 86 -- .../linkis/storage/utils/StorageHelper.scala | 107 -- .../apache/linkis/storage/utils/StorageUtils.scala | 229 ---- .../apache/linkis/storage/LineMetaDataTest.java | 36 + .../StorageResultSetWriterFactoryTest.java | 53 + .../script/writer/StorageScriptFsWriterTest.java | 8 +- .../storage/source/ResultsetFileSourceTest.java | 50 + .../src/test/resources/result-read-test.dolphin | 1 + .../src/test/resources/storage-read-test.dolphin | 1 + .../storage/utils/StorageConfigurationTest.scala | 3 +- .../linkis/cli/application/constants/AppKeys.java | 2 + .../cli/application/constants/LinkisKeys.java | 1 + .../command/template/UniversalCmdTemplate.java | 9 + .../application/operator/ujes/UJESConstants.java | 5 +- .../application/LinkisClientApplicationTest.java | 8 +- .../linkis/cli/core/constants/CommonKeys.java | 20 - .../linkis-computation-client/pom.xml | 4 - .../client/once/simple/SimpleOnceJob.scala | 7 +- .../ujes/client/response/JobProgressResult.scala | 17 +- .../ujes/client/response/OpenLogResult.scala | 2 +- .../computation/client/InteractiveJobTest.java | 3 +- .../governance/common/utils/ECPathUtils.java | 53 + .../governance/common/utils/LoggerUtils.java | 43 + .../common/conf/GovernaceCommonConf.scala | 73 -- .../common/conf/GovernanceCommonConf.scala | 87 ++ .../exception/GovernanceErrorException.scala | 5 +- .../governance/common/utils/ECPathUtils.scala | 48 - .../common/utils/GovernanceConstant.scala | 1 - .../governance/common/utils/GovernanceUtils.scala | 49 + .../linkis/governance/common/utils/JobUtils.scala | 10 +- .../common/conf/GovernanceCommonConfTest.scala | 4 +- .../common/utils/GovernanceConstantTest.scala | 2 - .../ecm/core/launch/ProcessEngineConnLaunch.scala | 9 +- .../linkis/ecm/core/report/NodeHealthReport.scala | 4 +- .../apache/linkis/ecm/core/utils/PortUtils.scala | 2 +- .../linkis-engineconn-manager-server/pom.xml | 7 - .../ecm/server/operator/EngineConnLogOperator.java | 219 ++++ .../server/operator/EngineConnYarnLogOperator.java | 147 +++ .../service/impl/DefaultEngineConnKillService.java | 285 +++++ .../linkis/ecm/server/conf/ECMConfiguration.scala | 13 + .../ecm/server/exception/ECMErrorException.scala | 11 +- .../linkis/ecm/server/hook/JarUDFLoadECMHook.scala | 1 - .../linkis/ecm/server/listener/ECMReadyEvent.scala | 17 +- .../server/operator/EngineConnLogOperator.scala | 241 ----- .../operator/EngineConnYarnLogOperator.scala | 124 --- .../ecm/server/report/DefaultECMHealthReport.scala | 4 +- .../ecm/server/service/ECMMetricsService.scala | 20 - .../ecm/server/service/EngineConnListService.scala | 47 - .../service/EngineConnPidCallbackService.scala | 26 - .../service/EngineConnStatusCallbackService.scala | 26 - .../ecm/server/service/LogCallbackService.scala | 26 - .../ecm/server/service/YarnCallbackService.scala | 28 - .../impl/AbstractEngineConnLaunchService.scala | 99 +- .../impl/BmlResourceLocalizationService.scala | 2 - .../service/impl/DefaultECMHealthService.scala | 9 +- .../service/impl/DefaultECMMetricsService.scala | 25 - .../service/impl/DefaultECMRegisterService.scala | 34 +- .../service/impl/DefaultEngineConnKillService.java | 242 ----- .../impl/DefaultEngineConnListService.scala | 192 ---- .../impl/DefaultEngineConnPidCallbackService.scala | 39 - .../DefaultEngineConnStatusCallbackService.scala | 67 -- .../service/impl/DefaultOperateService.scala | 24 +- .../service/impl/DefaultYarnCallbackService.scala | 40 - .../server/service/impl/ECMListenerService.scala | 55 + .../impl/ProcessEngineConnLaunchService.scala | 166 ++- .../ecm/server/spring/ECMSpringConfiguration.scala | 57 +- .../impl/DefaultECMRegisterServiceTest.java | 55 + .../operator/OperableOnceEngineConnOperator.java | 92 ++ .../executor/OnceExecutorExecutionContext.scala | 8 +- .../once/executor/hook/OnceEngineConnHook.scala | 5 - .../operator/OperableOnceEngineConnOperator.scala | 72 -- .../concurrent/monitor/HardwareMonitorService.java | 2 +- .../executor/async/AsyncEngineConnJob.scala | 10 + .../executor/async/AsyncExecuteExecutor.scala | 16 +- .../computation/executor/cs/CSTableRegister.scala | 2 +- .../executor/cs/CSTableResultSetWriter.scala | 2 +- .../executor/execute/ComputationExecutor.scala | 14 +- .../executor/execute/EngineExecutionContext.scala | 21 +- .../executor/hook/ComputationEngineConnHook.scala | 5 - .../computation/executor/hook/InitSQLHook.scala | 3 +- .../executor/hook/UDFLoadEngineConnHook.scala | 3 +- .../executor/hook/UseDatabaseEngineHook.scala | 2 +- .../service/TaskExecutionServiceImpl.scala | 311 +++--- .../access/ECTaskEntranceInfoAccessHelper.scala | 96 -- .../common/password/CommandPasswordCallback.java | 75 ++ .../common/password/StaticPasswordCallback.java | 40 + .../engineconn/common/conf/EngineConnConf.scala | 2 +- .../common/conf/EngineConnConstant.scala | 4 - .../engineconn/core/util/EngineConnUtils.scala | 8 +- .../engineconn/launch/EngineConnServer.scala | 1 + .../acessible/executor/log/AbstractLogCache.java | 38 - .../acessible/executor/log/MountLogCache.java | 2 +- .../acessible/executor/log/SendAppender.java | 6 +- .../acessible/executor/log/TimeLogCache.java | 41 - .../impl/EngineConnApplicationInfoOperator.java | 56 + .../conf/AccessibleExecutorConfiguration.scala | 12 + .../execution/AccessibleEngineConnExecution.scala | 2 +- .../executor/lock/EngineConnTimedLock.scala | 16 +- .../acessible/executor/log/LogHelper.scala | 3 - .../impl/EngineConnApplicationInfoOperator.scala | 50 - .../service/DefaultAccessibleService.scala | 41 +- .../executor/service/DefaultManagerService.scala | 3 +- .../executor/service/DefaultOperateService.scala | 24 +- .../service/EngineConnTimedLockService.scala | 12 +- .../acessible/executor/service/LockService.scala | 7 +- .../executor/utils/AccessableExecutorUtils.scala | 20 - .../callback/hook/CallbackEngineConnHook.scala | 36 +- .../service/EngineConnAfterStartCallback.scala | 5 +- .../callback/service/EngineConnCallback.scala | 22 +- .../callback/service/EngineConnPidCallback.scala | 4 +- .../callback/service/EngineConnTimedCallback.scala | 20 - .../errorcode/EngineconnCoreErrorCodeSummary.java | 6 +- .../exception/EngineConnPluginErrorCode.scala | 26 - .../exception/EngineConnPluginErrorException.scala | 7 +- .../JavaProcessEngineConnLaunchBuilder.scala | 2 +- .../apache/linkis/entrance/parser/ParserUtils.java | 2 +- .../persistence/QueryPersistenceManager.java | 6 + .../entrance/restful/EntranceRestfulApi.java | 2 +- .../entrance/server/DefaultEntranceServer.java | 3 +- .../apache/linkis/entrance/EntranceServer.scala | 6 + .../entrance/execute/DefaultEntranceExecutor.scala | 5 + .../linkis/entrance/execute/EntranceExecutor.scala | 14 +- .../entrance/execute/EntranceExecutorManager.scala | 2 +- .../interceptor/impl/TenantLabelSetUtils.scala | 5 + .../entrance/parser/CommonEntranceParser.scala | 25 +- .../persistence/EntranceResultSetEngine.scala | 2 +- .../linkis-jdbc-driver/pom.xml | 9 + .../linkis/ujes/jdbc/LinkisSQLErrorCode.java | 62 ++ .../org/apache/linkis/ujes/jdbc/UJESSQLDriver.java | 2 +- .../apache/linkis/ujes/jdbc/UJESSQLErrorCode.java | 57 - .../apache/linkis/ujes/jdbc/utils/JDBCUtils.java | 2 - .../linkis/ujes/jdbc/LinkisSQLConnection.scala | 465 ++++++++ .../linkis/ujes/jdbc/LinkisSQLException.scala | 37 + .../ujes/jdbc/LinkisSQLPreparedStatement.scala | 351 +++++++ .../linkis/ujes/jdbc/LinkisSQLStatement.scala | 331 ++++++ .../linkis/ujes/jdbc/UJESClientFactory.scala | 22 +- .../linkis/ujes/jdbc/UJESSQLConnection.scala | 340 ------ .../linkis/ujes/jdbc/UJESSQLDatabaseMetaData.scala | 202 ++-- .../linkis/ujes/jdbc/UJESSQLDriverMain.scala | 17 +- .../apache/linkis/ujes/jdbc/UJESSQLException.scala | 42 - .../ujes/jdbc/UJESSQLPreparedStatement.scala | 351 ------- .../apache/linkis/ujes/jdbc/UJESSQLResultSet.scala | 370 +++---- .../ujes/jdbc/UJESSQLResultSetMetaData.scala | 26 +- .../apache/linkis/ujes/jdbc/UJESSQLStatement.scala | 302 ------ .../linkis/ujes/jdbc/UJESSQLTypeParser.scala | 17 +- .../jdbc/hook/JDBCDriverPreExecutionHook.scala | 4 +- .../ujes/jdbc/hook/impl/NoLimitExecutionHook.scala | 6 +- .../jdbc/hook/impl/TableauPreExecutionHook.scala | 5 +- .../apache/linkis/ujes/jdbc/CreateConnection.java | 6 +- .../org/apache/linkis/ujes/jdbc/JDBCSpiTest.java | 12 +- .../linkis/ujes/jdbc/LinkisSQLStatementTest.java | 155 +++ .../ujes/jdbc/UJESSQLDatabaseMetaDataTest.java | 31 +- .../ujes/jdbc/UJESSQLPreparedStatementTest.java | 4 +- .../linkis/ujes/jdbc/UJESSQLResultSetTest.java | 4 +- .../linkis/ujes/jdbc/UJESSQLStatementTest.java | 155 --- .../linkis-application-manager/pom.xml | 12 +- .../loader/EngineConnPluginLoaderConf.java | 44 + .../loaders/CacheablesEngineConnPluginLoader.java | 3 +- .../loaders/DefaultEngineConnPluginLoader.java | 27 +- .../BmlEngineConnPluginResourceLoader.java | 4 +- .../loader/utils/EngineConnPluginUtils.java | 2 +- .../server/conf/EngineConnPluginConfiguration.java | 42 + .../interceptor/EngineConnLaunchInterceptor.java | 30 + .../loader/EngineConnPluginsLoaderFactory.java | 31 + .../AbstractEngineConnBmlResourceGenerator.java | 134 +++ .../DefaultEngineConnBmlResourceGenerator.java | 103 ++ .../localize/EngineConnBmlResourceGenerator.java | 30 + .../localize/EngineConnLocalizeResource.java | 30 + .../localize/EngineConnLocalizeResourceImpl.java | 69 ++ .../service/DefaultEngineConnLaunchService.java | 106 ++ .../DefaultEngineConnResourceFactoryService.java | 80 ++ .../service/DefaultEngineConnResourceService.java | 351 +++++++ .../server/service/EngineConnLaunchService.java | 26 + .../service/EngineConnResourceFactoryService.java | 30 + .../server/service/EngineConnResourceRequest.java | 53 + .../server/service/EngineConnResourceService.java | 43 + .../service/GetEngineConnResourceRequest.java | 25 + .../RefreshAllEngineConnResourceRequest.java | 28 + .../service/RefreshEngineConnResourceRequest.java | 25 + .../service/impl/EnginePluginAdminServiceImpl.java | 7 +- .../linkis/manager/LinkisManagerApplication.java | 29 + .../manager/am/LinkisManagerApplication.java | 29 - .../linkis/manager/am/conf/AMConfiguration.java | 125 +++ .../ApplicationManagerSpringConfiguration.java | 32 + .../manager/am/conf/ConfigurationMapCache.java | 69 ++ .../DefaultEngineConnConfigurationService.java | 72 ++ .../am/conf/EngineConnConfigurationService.java | 28 + .../linkis/manager/am/conf/ManagerMonitorConf.java | 45 + .../am/converter/DefaultMetricsConverter.java | 140 +++ .../event/message/EngineConnPidCallbackEvent.java | 36 + .../manager/am/exception/AMErrorException.java | 10 +- .../linkis/manager/am/label/AMLabelChecker.java | 56 + .../linkis/manager/am/label/AMLabelFilter.java | 59 ++ .../am/label/DefaultManagerLabelService.java | 63 ++ .../manager/am/label/EngineReuseLabelChooser.java | 27 + .../am/label/MultiUserEngineReuseLabelChooser.java | 104 ++ .../manager/am/locker/DefaultEngineNodeLocker.java | 73 ++ .../linkis/manager/am/locker/EngineNodeLocker.java | 29 + .../manager/am/manager/DefaultEMNodeManager.java | 218 ++++ .../am/manager/DefaultEngineNodeManager.java | 352 +++++++ .../linkis/manager/am/manager/EMNodeManager.java | 73 ++ .../manager/am/manager/EngineNodeManager.java | 67 ++ .../manager/am/pointer/AbstractNodePointer.java | 98 ++ .../manager/am/pointer/DefaultEMNodPointer.java | 111 ++ .../am/pointer/DefaultEngineNodPointer.java | 82 ++ .../am/pointer/DefaultNodePointerBuilder.java | 39 + .../manager/am/recycle/AssignNodeRuleExecutor.java | 42 + .../manager/am/recycle/RecyclingRuleExecutor.java | 28 + .../am/restful/ECResourceInfoRestfulApi.java | 2 +- .../linkis/manager/am/restful/EMRestfulApi.java | 57 +- .../manager/am/restful/EngineRestfulApi.java | 4 +- .../am/selector/DefaultECAvailableRule.java | 43 + .../manager/am/selector/DefaultNodeSelector.java | 75 ++ .../manager/am/selector/ECAvailableRule.java | 33 + .../linkis/manager/am/selector/NodeSelector.java | 32 + .../am/selector/rule/AvailableNodeSelectRule.java | 68 ++ .../selector/rule/ConcurrencyNodeSelectRule.java | 37 + .../am/selector/rule/HotspotExclusionRule.java | 62 ++ .../am/selector/rule/NewECMStandbyRule.java | 82 ++ .../manager/am/selector/rule/NodeSelectRule.java | 25 + .../am/selector/rule/ResourceNodeSelectRule.java | 120 +++ .../am/selector/rule/ScoreNodeSelectRule.java | 70 ++ .../am/selector/rule/TaskInfoNodeSelectRule.java | 73 ++ .../linkis/manager/am/service/EMEngineService.java | 40 + .../am/service/EngineConnPidCallbackService.java | 25 + .../service/EngineConnStatusCallbackService.java | 25 + .../linkis/manager/am/service/EngineService.java | 27 + .../manager/am/service/HeartbeatService.java | 25 + .../cache/ConfCacheRemoveBroadcastListener.java | 75 ++ .../am/service/em/DefaultECMOperateService.java | 37 + .../am/service/em/DefaultEMEngineService.java | 231 ++++ .../am/service/em/DefaultEMInfoService.java | 142 +++ .../am/service/em/DefaultEMRegisterService.java | 124 +++ .../am/service/em/DefaultEMUnregisterService.java | 79 ++ .../manager/am/service/em/ECMOperateService.java | 27 + .../manager/am/service/em/EMInfoService.java | 34 + .../manager/am/service/em/EMRegisterService.java | 26 + .../manager/am/service/em/EMUnregisterService.java | 28 + .../am/service/engine/AbstractEngineService.java | 41 + .../engine/DefaultEngineAskEngineService.java | 202 ++++ .../engine/DefaultEngineConnCanKillService.java | 196 ++++ .../service/engine/DefaultEngineCreateService.java | 431 ++++++++ .../service/engine/DefaultEngineInfoService.java | 145 +++ .../engine/DefaultEngineOperateService.java | 39 + .../engine/DefaultEngineRecycleService.java | 92 ++ .../service/engine/DefaultEngineReuseService.java | 265 +++++ .../service/engine/DefaultEngineStopService.java | 316 ++++++ .../service/engine/DefaultEngineSwitchService.java | 34 + .../am/service/engine/EngineAskEngineService.java | 26 + .../service/engine/EngineConnCanKillService.java | 26 + .../engine/EngineConnStatusCallbackService.java | 25 + .../am/service/engine/EngineCreateService.java | 29 + .../am/service/engine/EngineInfoService.java | 52 + .../am/service/engine/EngineOperateService.java | 28 + .../am/service/engine/EngineRecycleService.java | 28 + .../am/service/engine/EngineReuseService.java | 29 + .../am/service/engine/EngineStopService.java | 61 ++ .../am/service/engine/EngineSwitchService.java | 27 + .../am/service/heartbeat/AMHeartbeatService.java | 86 ++ .../impl/DefaultEngineConnPidCallbackService.java | 61 ++ .../DefaultEngineConnStatusCallbackService.java | 129 +++ .../am/service/impl/ECResourceInfoServiceImpl.java | 8 +- .../am/service/monitor/NodeHeartbeatMonitor.java | 372 +++++++ .../manager/am/util/ECResourceInfoUtils.java | 2 +- .../apache/linkis/manager/am/util/LinkisUtils.java | 248 +++++ .../apache/linkis/manager/am/utils/AMUtils.java | 338 ++++++ .../manager/am/utils/DefaultRetryHandler.java | 122 +++ .../linkis/manager/am/utils/RetryHandler.java | 45 + .../linkis/manager/am/vo/AMEngineNodeVo.java | 6 +- .../org/apache/linkis/manager/am/vo/EMNodeVo.java | 6 +- .../linkis/manager/label/LabelManagerUtils.java | 37 + .../manager/label/conf/LabelManagerConf.java | 26 + .../label/score/DefaultNodeLabelScorer.java | 203 ++++ .../label/score/LabelScoreServiceInstance.java | 51 + .../manager/label/score/NodeLabelScorer.java | 41 + .../manager/label/service/NodeLabelAddService.java | 28 + .../label/service/NodeLabelRemoveService.java | 25 + .../manager/label/service/NodeLabelService.java | 83 ++ .../label/service/ResourceLabelService.java | 62 ++ .../manager/label/service/UserLabelService.java | 37 + .../service/impl/DefaultNodeLabelAddService.java | 69 ++ .../impl/DefaultNodeLabelRemoveService.java | 82 ++ .../service/impl/DefaultNodeLabelService.java | 459 ++++++++ .../service/impl/DefaultResourceLabelService.java | 161 +++ .../service/impl/DefaultUserLabelService.java | 167 +++ .../linkis/manager/rm/conf/ResourceStatus.java | 4 +- .../linkis/manager/rm/domain/RMLabelContainer.java | 11 +- .../manager/rm/entity/LabelResourceMapping.java | 68 ++ .../manager/rm/entity/ResourceOperationType.java | 25 + .../service/impl/ExternalResourceServiceImpl.java | 10 +- .../manager/rm/external/yarn/YarnAppInfo.java | 78 ++ .../rm/external/yarn/YarnResourceRequester.java | 454 ++++++++ .../manager/rm/message/RMMessageService.java | 75 ++ .../linkis/manager/rm/restful/RMMonitorRest.java | 880 ++++++++++++++++ .../manager/rm/service/LabelResourceService.java | 51 + .../manager/rm/service/RequestResourceService.java | 422 ++++++++ .../manager/rm/service/ResourceLockService.java | 95 ++ .../linkis/manager/rm/service/ResourceManager.java | 96 ++ .../linkis/manager/rm/service/impl/ChangeType.java | 40 + .../rm/service/impl/DefaultReqResourceService.java | 37 + .../rm/service/impl/DefaultResourceManager.java | 894 ++++++++++++++++ .../impl/DriverAndYarnReqResourceService.java | 151 +++ .../rm/service/impl/LabelResourceServiceImpl.java | 102 ++ .../rm/service/impl/ResourceLogService.java | 222 ++++ .../rm/service/impl/UserResourceService.java | 128 +++ .../apache/linkis/manager/rm/utils/RMUtils.java | 288 +++++ .../manager/rm/utils/RequestKerberosUrlUtils.java | 6 - .../linkis/manager/rm/utils/UserConfiguration.java | 210 ++++ .../manager/service/common/label/LabelChecker.java | 31 + .../manager/service/common/label/LabelFilter.java | 29 + .../service/common/label/ManagerLabelService.java | 32 + .../service/common/metrics/MetricsConverter.java | 55 + .../service/common/pointer/EMNodPointer.java | 33 + .../service/common/pointer/EngineNodePointer.java | 34 + .../service/common/pointer/NodePointer.java | 64 ++ .../service/common/pointer/NodePointerBuilder.java | 28 + .../mapper/common/EngineConnBmlResourceMapper.xml | 44 +- .../common/ExternalResourceProviderDaoImpl.xml | 4 +- .../loader/EngineConnPluginLoaderConf.scala | 50 - .../conf/EngineConnPluginConfiguration.scala | 42 - .../interceptor/EngineConnLaunchInterceptor.scala | 34 - .../server/loader/EngineConnPluginsLoader.scala | 32 - .../AbstractEngineConnBmlResourceGenerator.scala | 122 --- .../DefaultEngineConnBmlResourceGenerator.scala | 109 -- .../localize/EngineConnBmlResourceGenerator.scala | 44 - .../service/DefaultEngineConnLaunchService.scala | 90 -- .../DefaultEngineConnResourceFactoryService.scala | 64 -- .../service/DefaultEngineConnResourceService.scala | 294 ------ .../server/service/EngineConnLaunchService.scala | 31 - .../service/EngineConnResourceFactoryService.scala | 33 - .../server/service/EngineConnResourceService.scala | 78 -- .../linkis/manager/am/conf/AMConfiguration.scala | 118 --- .../manager/am/conf/ConfigurationMapCache.scala | 64 -- .../am/conf/EngineConnConfigurationService.scala | 70 -- .../manager/am/conf/ManagerMonitorConf.scala | 43 - .../am/converter/DefaultMetricsConverter.scala | 102 -- .../linkis/manager/am/label/AMLabelChecker.scala | 50 - .../linkis/manager/am/label/AMLabelFilter.scala | 56 - .../am/label/DefaultManagerLabelService.scala | 72 -- .../manager/am/label/EngineReuseLabelRester.scala | 28 - .../label/MultiUserEngineReuseLabelChooser.scala | 83 -- .../am/locker/DefaultEngineNodeLocker.scala | 61 -- .../manager/am/locker/EngineNodeLocker.scala | 28 - .../manager/am/manager/DefaultEMNodeManager.scala | 193 ---- .../am/manager/DefaultEngineNodeManager.scala | 311 ------ .../linkis/manager/am/manager/EMNodeManager.scala | 71 -- .../manager/am/manager/EngineNodeManager.scala | 82 -- .../manager/am/pointer/AbstractNodePointer.scala | 83 -- .../manager/am/pointer/DefaultEMNodPointer.scala | 83 -- .../am/pointer/DefaultEngineNodPointer.scala | 68 -- .../am/pointer/DefaultNodePointerBuilder.scala | 40 - .../am/recycle/AssignNodeRuleExecutor.scala | 38 - .../manager/am/recycle/RecyclingRuleExecutor.scala | 29 - .../linkis/manager/am/rpc/ManagerRPCFormats.scala | 35 - .../manager/am/selector/DefaultNodeSelector.scala | 75 -- .../manager/am/selector/ECAvailableRule.scala | 55 - .../linkis/manager/am/selector/NodeSelector.scala | 31 - .../am/selector/rule/AvailableNodeSelectRule.scala | 60 -- .../selector/rule/ConcurrencyNodeSelectRule.scala | 35 - .../am/selector/rule/HotspotExclusionRule.scala | 70 -- .../am/selector/rule/NewECMStandbyRule.scala | 83 -- .../manager/am/selector/rule/NodeSelectRule.scala | 26 - .../am/selector/rule/OverLoadNodeSelectRule.scala | 66 -- .../am/selector/rule/ResourceNodeSelectRule.scala | 101 -- .../am/selector/rule/ScoreNodeSelectRule.scala | 55 - .../am/selector/rule/TaskInfoNodeSelectRule.scala | 63 -- .../manager/am/service/EMEngineService.scala | 39 - .../linkis/manager/am/service/EngineService.scala | 28 - .../manager/am/service/HeartbeatService.scala | 26 - .../cache/ConfCacheRemoveBroadcastListener.scala | 70 -- .../am/service/em/DefaultECMOperateService.scala | 36 - .../am/service/em/DefaultEMEngineService.scala | 146 --- .../am/service/em/DefaultEMInfoService.scala | 124 --- .../am/service/em/DefaultEMRegisterService.scala | 121 --- .../am/service/em/DefaultEMUnregisterService.scala | 89 -- .../manager/am/service/em/ECMOperateService.scala | 27 - .../manager/am/service/em/EMInfoService.scala | 36 - .../manager/am/service/em/EMRegisterService.scala | 33 - .../am/service/em/EMUnregisterService.scala | 29 - .../am/service/engine/AbstractEngineService.scala | 41 - .../engine/DefaultEngineAskEngineService.scala | 170 --- .../engine/DefaultEngineConnCanKillService.scala | 173 --- .../DefaultEngineConnStatusCallbackService.scala | 104 -- .../engine/DefaultEngineCreateService.scala | 405 ------- .../service/engine/DefaultEngineInfoService.scala | 145 --- .../engine/DefaultEngineOperateService.scala | 42 - .../engine/DefaultEngineRecycleService.scala | 82 -- .../service/engine/DefaultEngineReuseService.scala | 219 ---- .../service/engine/DefaultEngineStopService.scala | 288 ----- .../engine/DefaultEngineSwitchService.scala | 32 - .../am/service/engine/EngineAskEngineService.scala | 27 - .../service/engine/EngineConnCanKillService.scala | 26 - .../engine/EngineConnStatusCallbackService.scala | 26 - .../am/service/engine/EngineCreateService.scala | 31 - .../am/service/engine/EngineInfoService.scala | 50 - .../am/service/engine/EngineKillService.scala | 31 - .../am/service/engine/EngineOperateService.scala | 33 - .../am/service/engine/EngineRecycleService.scala | 29 - .../am/service/engine/EngineReuseService.scala | 30 - .../am/service/engine/EngineStopService.scala | 74 -- .../am/service/engine/EngineSwitchService.scala | 29 - .../am/service/heartbeat/AMHeartbeatService.scala | 83 -- .../am/service/monitor/NodeHeartbeatMonitor.scala | 493 --------- .../apache/linkis/manager/am/utils/AMUtils.scala | 303 ------ .../linkis/manager/label/LabelManagerUtils.scala | 37 - .../manager/label/conf/LabelManagerConf.scala | 27 - .../label/score/DefaultNodeLabelScorer.scala | 182 ---- .../label/score/LabelScoreServiceInstance.scala | 35 - .../manager/label/score/NodeLabelScorer.scala | 46 - .../label/service/NodeLabelAddService.scala | 28 - .../label/service/NodeLabelRemoveService.scala | 26 - .../manager/label/service/NodeLabelService.scala | 93 -- .../label/service/ResourceLabelService.scala | 68 -- .../manager/label/service/UserLabelService.scala | 38 - .../service/impl/DefaultNodeLabelAddService.scala | 56 - .../impl/DefaultNodeLabelRemoveService.scala | 75 -- .../service/impl/DefaultNodeLabelService.scala | 411 -------- .../service/impl/DefaultResourceLabelService.scala | 162 --- .../service/impl/DefaultUserLabelService.scala | 122 --- .../manager/rm/entity/LabelResourceMap.scala | 52 - .../manager/rm/entity/ResourceOperationType.scala | 26 - .../manager/rm/external/yarn/YarnAppInfo.scala | 29 - .../rm/external/yarn/YarnResourceRequester.scala | 405 ------- .../manager/rm/message/RMMessageService.scala | 68 -- .../linkis/manager/rm/restful/RMMonitorRest.scala | 783 -------------- .../manager/rm/service/LabelResourceService.scala | 50 - .../rm/service/RequestResourceService.scala | 399 ------- .../manager/rm/service/ResourceLockService.scala | 80 -- .../manager/rm/service/ResourceManager.scala | 99 -- .../service/impl/DefaultReqResourceService.scala | 32 - .../rm/service/impl/DefaultResourceManager.scala | 889 ---------------- .../impl/DriverAndYarnReqResourceService.scala | 125 --- .../rm/service/impl/LabelResourceServiceImpl.scala | 100 -- .../rm/service/impl/ResourceLogService.scala | 240 ----- .../rm/service/impl/UserResourceService.scala | 114 -- .../apache/linkis/manager/rm/utils/RMUtils.scala | 247 ----- .../manager/rm/utils/UserConfiguration.scala | 174 --- .../service/common/label/LabelChecker.scala | 31 - .../manager/service/common/label/LabelFilter.scala | 30 - .../service/common/label/ManagerLabelService.scala | 33 - .../service/common/metrics/MetricsConverter.scala | 57 - .../service/common/pointer/EMNodPointer.scala | 33 - .../service/common/pointer/EngineNodePointer.scala | 34 - .../service/common/pointer/NodePointer.scala | 70 -- .../common/pointer/NodePointerBuilder.scala | 28 - .../src/test/java/org/apache/linkis/Scan.java | 26 + .../org/apache/linkis/WebApplicationServer.java | 34 + .../engineplugin/server/dao/BaseDaoTest.java | 31 + .../server/dao/EngineConnBmlResourceDaoTest.java | 111 ++ .../am/util/ECResourceInfoLinkisUtilsTest.java | 52 + .../dao/ExternalResourceProviderDaoTest.java | 40 + .../src/test/resources/application.properties | 36 + .../src/test/resources/create.sql | 47 + .../src/test/resources/create_pg.sql | 48 + .../src/test/resources/linkis.properties | 25 + .../manager/label/conf/LabelCommonConfig.java | 4 +- .../manager/label/entity/cluster/ClusterLabel.java | 17 +- .../manager/label/entity/engine/EngineType.scala | 2 + .../label/entity/engine/EngineTypeLabel.java | 30 + .../manager/label/entity/engine/RunType.scala | 3 +- .../label/exception/LabelErrorException.java | 10 +- .../linkis/manager/label/utils/LabelUtils.java | 3 +- .../linkis/manager/label/TestLabelBuilder.java | 5 +- .../label/entity/engine/EngineTypeLabelTest.java | 46 +- .../linkis-manager/linkis-manager-common/pom.xml | 14 - .../manager/common/conf/ManagerCommonConf.java | 35 + .../manager/common/conf/RMConfiguration.java | 114 ++ .../manager/common/entity/node/AMEMNode.java | 29 +- .../manager/common/entity/node/AMEngineNode.java | 19 +- .../manager/common/entity/node/EngineNode.java | 4 - .../manager/common/entity/node/InfoRMNode.java | 23 + .../manager/common/entity/node/LabelNode.java | 4 +- .../linkis/manager/common/entity/node/Node.java | 8 + .../entity/persistence/ECResourceInfoRecord.java | 16 + .../common/entity/persistence/PersistenceNode.java | 20 + .../entity/persistence/PersistenceNodeEntity.java | 23 + .../entity/persistence/PersistenceNodeMetrics.java | 5 +- .../common/entity/recycle/AssignEMNodeRule.java | 41 + .../common/entity/recycle/AssignNodeRule.java | 41 + .../common/entity/recycle/AssignUserRule.java | 32 + .../common/entity/recycle/RecyclingRule.java | 23 + .../common/entity/resource/CPUResource.java | 132 +++ .../entity/resource/DriverAndYarnResource.java | 222 ++++ .../common/entity/resource/InstanceResource.java | 54 + .../entity/resource/LoadInstanceResource.java | 161 +++ .../common/entity/resource/LoadResource.java | 129 +++ .../common/entity/resource/MemoryResource.java | 120 +++ .../manager/common/entity/resource/Resource.java | 134 +++ .../common/entity/resource/SpecialResource.java | 355 +++++++ .../entity/resource/UserAvailableResource.java | 36 + .../common/entity/resource/YarnResource.java | 176 ++++ .../manager/common/exception/RMErrorException.java | 10 +- .../manager/common/monitor/ManagerMonitor.java | 20 + .../linkis/manager/common/operator/Operator.java | 56 + .../manager/common/operator/OperatorFactory.java | 33 + .../common/operator/OperatorFactoryImpl.java | 72 ++ .../linkis/manager/common/protocol/EngineLock.java | 22 + .../manager/common/protocol/OperateRequest.java | 39 + .../manager/common/protocol/OperateResponse.java | 28 + .../manager/common/protocol/RequestEngineLock.java | 48 + .../common/protocol/RequestEngineUnlock.java | 36 + .../common/protocol/RequestManagerUnlock.java | 59 ++ .../common/protocol/ResponseEngineLock.java | 56 + .../common/protocol/ResponseEngineUnlock.java | 36 + .../manager/common/protocol/bml/BmlResource.java | 9 - .../common/protocol/em/ECMOperateRequest.java | 51 + .../common/protocol/em/ECMOperateResponse.java | 52 + .../protocol/em/EMResourceRegisterRequest.java | 79 ++ .../common/protocol/em/RegisterEMRequest.java | 79 ++ .../common/protocol/em/RegisterEMResponse.java | 50 + .../protocol/engine/EngineAskAsyncResponse.java | 48 + .../protocol/engine/EngineAsyncResponse.java | 24 + .../protocol/engine/EngineConnStatusCallback.java | 55 + .../engine/EngineConnStatusCallbackToAM.java | 69 ++ .../common/protocol/engine/EngineCreateError.java | 61 ++ .../protocol/engine/EngineCreateSuccess.java | 48 + .../protocol/engine/EngineOperateRequest.java | 42 + .../protocol/engine/EngineOperateResponse.java | 53 + .../common/protocol/engine/EngineStopRequest.java | 51 +- .../common/protocol/label/LabelReportRequest.java | 43 + .../common/protocol/label/LabelUpdateRequest.java | 36 + .../common/protocol/resource/ResourceProtocol.java | 22 + .../protocol/resource/ResourceUsedProtocol.java | 51 + .../protocol/resource/ResourceWithStatus.java | 103 ++ .../protocol/resource/ResponseTaskRunningInfo.java | 74 ++ .../resource/ResponseTaskYarnResource.java | 41 + .../linkis/manager/common/utils/ManagerUtils.java | 7 +- .../linkis/manager/common/utils/ResourceUtils.java | 312 ++++++ .../linkis/manager/rm/AvailableResource.java | 34 + .../linkis/manager/rm/NotEnoughResource.java | 36 + .../linkis/manager/rm/RequestResourceAndWait.java | 47 + .../org/apache/linkis/manager/rm/ResourceInfo.java | 34 + .../apache/linkis/manager/rm/ResultResource.java | 20 + .../manager/common/conf/ManagerCommonConf.scala | 30 - .../manager/common/conf/RMConfiguration.scala | 91 -- .../common/entity/recycle/RecyclingRule.scala | 32 - .../manager/common/entity/resource/Resource.scala | 848 --------------- .../manager/common/monitor/ManagerMonitor.scala | 20 - .../linkis/manager/common/operator/Operator.scala | 44 - .../manager/common/operator/OperatorFactory.scala | 64 -- .../manager/common/protocol/EngineLock.scala | 39 - .../manager/common/protocol/OperateRequest.scala | 45 - .../manager/common/protocol/OperateResponse.scala | 25 - .../common/protocol/ServiceHealthReport.scala | 27 - .../manager/common/protocol/ServiceState.scala | 20 - .../common/protocol/em/ECMOperateRequest.scala | 37 - .../common/protocol/em/ECMOperateResponse.scala | 32 - .../protocol/em/EMResourceRegisterRequest.scala | 67 -- .../common/protocol/em/RegisterEMRequest.scala | 67 -- .../common/protocol/em/RegisterEMResponse.scala | 24 - .../protocol/engine/EngineAsyncResponse.scala | 35 - .../protocol/engine/EngineConnStatusCallback.scala | 55 - .../protocol/engine/EngineOperateRequest.scala | 31 - .../protocol/engine/EngineOperateResponse.scala | 32 - .../common/protocol/label/LabelUpdateRequest.scala | 27 - .../protocol/resource/ResourceProtocol.scala | 48 - .../protocol/resource/ResourceWithStatus.scala | 33 - .../common/serializer/NodeResourceSerializer.scala | 62 -- .../serializer/RegisterEMRequestSerializer.scala | 58 - .../manager/common/utils/ResourceUtils.scala | 218 ---- .../linkis/manager/rm/ResourceManagerParam.scala | 51 - .../apache/linkis/manager/rm/ResultResource.scala | 46 - .../linkis/manager/dao/NodeManagerMapper.java | 4 +- .../exception/PersistenceErrorException.java | 10 +- .../persistence/NodeManagerPersistence.java | 20 +- .../persistence/NodeMetricManagerPersistence.java | 12 +- .../persistence/ResourceManagerPersistence.java | 6 +- .../impl/DefaultNodeManagerPersistence.java | 61 +- .../impl/DefaultNodeMetricManagerPersistence.java | 27 +- .../impl/DefaultResourceLabelPersistence.java | 6 +- .../impl/DefaultResourceManagerPersistence.java | 5 +- .../mapper/common/ECResourceRecordMapper.xml | 107 -- .../resources/mapper/common/LabelManagerMapper.xml | 541 ---------- .../resources/mapper/common/LockManagerMapper.xml | 16 +- .../resources/mapper/common/NodeManagerMapper.xml | 147 ++- .../mapper/common/NodeMetricManagerMapper.xml | 51 +- .../mapper/common/ResourceManagerMapper.xml | 101 -- .../mapper/mysql/ECResourceRecordMapper.xml | 129 +++ .../resources/mapper/mysql/LabelManagerMapper.xml | 585 +++++++++++ .../mapper/mysql/ResourceManagerMapper.xml | 148 +++ .../mapper/postgresql/ECResourceRecordMapper.xml | 127 +++ .../mapper/postgresql/LabelManagerMapper.xml | 540 ++++++++++ .../mapper/postgresql/ResourceManagerMapper.xml | 104 ++ .../manager/dao/ECResourceRecordMapperTest.java | 134 +++ .../linkis/manager/dao/LabelManagerMapperTest.java | 293 ++++++ .../linkis/manager/dao/NodeManagerMapperTest.java | 3 +- .../manager/dao/ResourceManagerMapperTest.java | 45 +- .../src/test/resources/application.properties | 4 + .../src/test/resources/create.sql | 53 +- .../src/test/resources/create_pg.sql | 189 ++++ linkis-dist/bin/checkEnv.sh | 4 +- linkis-dist/bin/install-linkis-to-kubernetes.sh | 81 +- linkis-dist/bin/install.sh | 220 ++-- linkis-dist/deploy-config/db.sh | 7 + linkis-dist/deploy-config/linkis-env.sh | 10 +- linkis-dist/docker/ldh.Dockerfile | 8 +- .../licenses/LICENSE-spark-2.4.3-bin-hadoop2.7.txt | 4 - linkis-dist/docker/scripts/entry-point-ldh.sh | 5 +- linkis-dist/docker/scripts/prepare-ldh-image.sh | 8 +- linkis-dist/helm/README.md | 20 +- linkis-dist/helm/README_CN.md | 20 +- .../linkis/templates/configmap-init-sql.yaml | 5 +- .../linkis/templates/configmap-linkis-config.yaml | 49 +- linkis-dist/helm/charts/linkis/templates/jobs.yaml | 24 +- .../templates/linkis-cg-engineconnmanager.yaml | 2 +- .../linkis/templates/linkis-cg-entrance.yaml | 2 +- .../linkis/templates/linkis-cg-linkismanager.yaml | 2 +- .../charts/linkis/templates/linkis-mg-eureka.yaml | 6 +- .../charts/linkis/templates/linkis-mg-gateway.yaml | 2 +- .../linkis/templates/linkis-ps-publicservice.yaml | 2 +- linkis-dist/helm/charts/linkis/values.yaml | 6 +- linkis-dist/helm/scripts/common.sh | 2 +- .../helm/scripts/install-charts-with-ldh.sh | 5 +- linkis-dist/helm/scripts/install-charts.sh | 3 +- linkis-dist/helm/scripts/install-ldh.sh | 5 +- linkis-dist/helm/scripts/install-linkis.sh | 9 +- linkis-dist/helm/scripts/install-mysql.sh | 3 +- linkis-dist/helm/scripts/prepare-for-spark.sh | 8 +- linkis-dist/package/bin/linkis-cli-hive | 2 +- linkis-dist/package/bin/linkis-cli-spark-submit | 6 +- .../conf/linkis-cg-engineconnmanager.properties | 2 + .../conf/linkis-cg-linkismanager.properties | 5 +- .../package/conf/linkis-cli/linkis-cli.properties | 4 +- .../package/conf/linkis-mg-gateway.properties | 1 - .../conf/linkis-ps-publicservice.properties | 1 - linkis-dist/package/conf/linkis.properties | 19 +- linkis-dist/package/conf/log4j2.xml | 2 +- linkis-dist/package/db/linkis_ddl.sql | 2 + linkis-dist/package/db/linkis_ddl_pg.sql | 1109 ++++++++++++++++++++ linkis-dist/package/db/linkis_dml.sql | 76 +- linkis-dist/package/db/linkis_dml_pg.sql | 715 +++++++++++++ .../package/db/module/linkis_configuration_dml.sql | 14 +- linkis-dist/package/db/module/linkis_manager.sql | 1 + .../db/upgrade/1.4.0_schema/mysql/linkis_ddl.sql | 19 + .../db/upgrade/1.4.0_schema/mysql/linkis_dml.sql | 72 ++ .../package/sbin/ext/linkis-cg-linkismanager | 2 +- linkis-dist/package/sbin/ext/linkis-common-start | 4 +- linkis-dist/package/sbin/ext/linkis-mg-eureka | 4 +- linkis-dist/package/sbin/ext/linkis-mg-gateway | 4 +- .../package/sbin/kill-ec-process-by-port.sh | 28 + linkis-dist/package/sbin/linkis-daemon.sh | 0 linkis-dist/pom.xml | 8 +- linkis-dist/release-docs/LICENSE | 6 +- .../licenses/LICENSE-aliyun-java-sdk-core.txt | 13 + .../licenses/LICENSE-aliyun-java-sdk-kms.txt | 13 + .../licenses/LICENSE-aliyun-java-sdk-ram.txt | 13 + .../licenses/LICENSE-aliyun-sdk-oss.txt | 201 ++++ .../licenses/LICENSE-aws-java-sdk-core.txt | 53 + .../licenses/LICENSE-aws-java-sdk-kms.txt | 53 + .../licenses/LICENSE-aws-java-sdk-s3.txt | 53 + .../licenses/LICENSE-hadoop-aliyun.txt | 270 +++++ .../release-docs/licenses/LICENSE-hazelcast.txt | 202 ++++ .../release-docs/licenses/LICENSE-ion-java.txt | 202 ++++ .../licenses/LICENSE-jmespath-java.txt | 53 + .../licenses/LICENSE-opentracing-api.txt | 201 ++++ .../licenses/LICENSE-opentracing-noop.txt | 201 ++++ .../licenses/LICENSE-opentracing-util.txt | 201 ++++ .../release-docs/licenses/LICENSE-postgresql.txt | 33 + linkis-engineconn-plugins/elasticsearch/pom.xml | 4 - .../ElasticSearchEngineConnPlugin.java | 89 ++ ...lasticSearchProcessEngineConnLaunchBuilder.java | 31 + .../conf/ElasticSearchConfiguration.java | 63 ++ .../conf/ElasticSearchEngineConsoleConf.java | 64 ++ .../exception/EsConvertResponseException.java | 27 + .../exception/EsParamsIllegalException.java | 27 + .../executor/ElasticSearchEngineConnExecutor.java | 285 +++++ .../executor/client/ElasticSearchExecutor.java | 27 + .../elasticsearch/executor/client/EsClient.java | 66 ++ .../executor/client/EsClientFactory.java | 220 ++++ .../executor/client/EsClientImpl.java | 106 ++ .../executor/client/EsClientOperate.java | 30 + .../client/impl/ElasticSearchExecutorImpl.java | 130 +++ .../elasticsearch/src/main/resources/log4j2.xml | 8 +- .../ElasticSearchEngineConnPlugin.scala | 74 -- ...asticSearchProcessEngineConnLaunchBuilder.scala | 30 - .../conf/ElasticSearchConfiguration.scala | 49 - .../conf/ElasticSearchEngineConsoleConf.scala | 49 - .../exception/EsConvertResponseException.scala | 24 - .../exception/EsEngineException.scala | 22 - .../exception/EsParamsIllegalException.scala | 24 - .../executor/ElasticSearchEngineConnExecutor.scala | 232 ---- .../executor/ElasticSearchExecutorOrder.scala | 25 - .../executor/client/ElasticSearchExecutor.scala | 46 - .../executor/client/ElasticSearchResponse.scala | 0 .../elasticsearch/executor/client/EsClient.scala | 131 --- .../executor/client/EsClientFactory.scala | 185 ---- .../executor/client/ResponseHandler.scala | 10 + .../client/impl/ElasticSearchExecutorImpl.scala | 112 -- .../executor/client/impl/ResponseHandlerImpl.scala | 22 +- linkis-engineconn-plugins/flink/pom.xml | 41 +- .../flink/errorcode/FlinkErrorCodeSummary.java | 4 +- .../flink/operator/TriggerSavepointOperator.java | 92 ++ .../flink/src/main/resources/log4j2.xml | 6 +- .../flink/config/FlinkEnvConfiguration.scala | 7 +- .../flink/executor/FlinkExecutor.scala | 6 +- .../flink/executor/FlinkOnceExecutor.scala | 16 +- .../interceptor/FlinkJobResCleanInterceptor.scala | 93 ++ .../interceptor/FlinkJobSubmitInterceptor.scala | 50 + .../factory/FlinkApplicationExecutorFactory.scala | 12 +- .../flink/factory/FlinkEngineConnFactory.scala | 1 - .../flink/operator/TriggerSavepointOperator.scala | 49 - .../resource/FlinkJobLocalResourceCleaner.scala | 69 ++ .../flink/resource/FlinkJobResourceCleaner.scala | 38 + linkis-engineconn-plugins/hive/pom.xml | 21 +- .../hive/serde/CustomerDelimitedJSONSerDe.java | 12 - .../main/resources/linkis-engineconn.properties | 5 +- .../hive/src/main/resources/log4j2.xml | 12 +- .../hive/conf/HiveEngineConfiguration.scala | 9 + .../hive/creation/HiveEngineConnFactory.scala | 119 ++- .../engineplugin/hive/entity/HiveSession.scala | 11 +- .../HiveEngineConcurrentConnExecutor.scala | 540 ++++++++++ .../hive/executor/HiveEngineConnExecutor.scala | 8 +- linkis-engineconn-plugins/impala/pom.xml | 143 +++ .../impala/src/main/assembly/distribution.xml | 70 ++ .../impala/client/ExecutionListener.java | 35 + .../engineplugin/impala/client/ImpalaClient.java | 77 ++ .../impala/client/ImpalaResultSet.java | 55 + .../client/exception/ImpalaEngineException.java | 52 + .../client/exception/ImpalaErrorCodeSummary.java | 51 + .../impala/client/protocol/ExecHandler.java | 72 ++ .../impala/client/protocol/ExecProgress.java | 47 + .../impala/client/protocol/ExecStatus.java | 72 ++ .../impala/client/protocol/ExecSummary.java | 45 + .../impala/client/protocol/QueryColumn.java | 39 + .../impala/client/thrift/ImpalaThriftClient.java | 278 +++++ .../client/thrift/ImpalaThriftExecution.java | 186 ++++ .../client/thrift/ImpalaThriftResultSetV7.java | 361 +++++++ .../impala/client/thrift/ImpalaThriftSession.java | 104 ++ .../client/thrift/ImpalaThriftSessionFactory.java | 196 ++++ .../impala/client/util/ThriftUtil.java | 65 ++ .../main/resources/linkis-engineconn.properties | 23 + .../impala/src/main/resources/log4j2.xml | 89 ++ .../impala/ImpalaEngineConnPlugin.scala | 66 ++ .../ImpalaProcessEngineConnLaunchBuilder.scala | 35 + .../impala/conf/ImpalaConfiguration.scala | 68 ++ .../impala/conf/ImpalaEngineConfig.scala | 48 + .../impala/executor/ImpalaEngineConnExecutor.scala | 510 +++++++++ .../impala/factory/ImpalaEngineConnFactory.scala | 44 + .../executer/TestImpalaEngineConnExecutor.scala | 144 +++ linkis-engineconn-plugins/io_file/pom.xml | 10 - .../io_file/src/main/resources/log4j2.xml | 8 +- .../io/executor/IoEngineConnExecutor.scala | 93 +- .../engineplugin/io/service/FsProxyService.scala | 18 +- .../manager/engineplugin/io/utils/IOHelp.scala | 34 +- .../src/test/resources/testIoResult.dolphin | 0 .../io/executor/IoEngineConnExecutorTest.java | 73 ++ linkis-engineconn-plugins/jdbc/pom.xml | 15 +- .../main/resources/linkis-engineconn.properties | 3 +- .../jdbc/src/main/resources/log4j2.xml | 6 +- .../jdbc/executor/JDBCEngineConnExecutor.scala | 14 +- .../engineplugin/jdbc/executor/JDBCHelper.java | 44 +- linkis-engineconn-plugins/openlookeng/pom.xml | 4 - .../openlookeng/OpenLooKengECPlugin.java | 85 ++ .../builder/OpenLooKengProcessECLaunchBuilder.java | 2 +- .../executor/OpenLooKengEngineConnExecutor.java | 4 +- .../openlookeng/src/main/resources/log4j2.xml | 6 +- .../openlookeng/OpenLooKengECPlugin.scala | 80 -- linkis-engineconn-plugins/pipeline/pom.xml | 10 - .../pipeline/src/main/resources/log4j2.xml | 6 +- linkis-engineconn-plugins/pom.xml | 8 + linkis-engineconn-plugins/presto/pom.xml | 4 - .../presto/PrestoEngineConnPlugin.java | 72 ++ .../PrestoProcessEngineConnLaunchBuilder.java | 30 + .../presto/conf/PrestoConfiguration.java | 63 ++ .../engineplugin/presto/conf/PrestoEngineConf.java | 53 + .../presto/exception/PrestoClientException.java | 27 + .../exception/PrestoStateInvalidException.java | 27 + .../presto/executor/PrestoEngineConnExecutor.java | 468 +++++++++ .../engineplugin/presto/utils/PrestoSQLHook.java | 34 + .../presto/src/main/resources/log4j2.xml | 8 +- .../presto/PrestoEngineConnPlugin.scala | 71 -- .../PrestoProcessEngineConnLaunchBuilder.scala | 30 - .../presto/conf/PrestoConfiguration.scala | 49 - .../presto/conf/PrestoEngineConf.scala | 46 - .../presto/exception/PrestoException.scala | 26 - .../presto/executor/PrestoEngineConnExecutor.scala | 453 -------- .../engineplugin/presto/utils/PrestoSQLHook.scala | 36 - .../presto/TestPrestoEngineConnPlugin.java | 30 + .../TestPrestoProcessEngineConnLaunchBuilder.java | 35 + .../presto/conf/TestPrestoConfiguration.java | 40 + .../presto/exception/TestPrestoException.java | 42 + .../factory/TestPrestoEngineConnFactory.java | 43 + .../presto/utils/TestPrestoSQLHook.java | 31 + .../presto/TestPrestoEngineConnPlugin.scala | 30 - .../TestPrestoProcessEngineConnLaunchBuilder.scala | 34 - .../presto/conf/TestPrestoConfiguration.scala | 42 - .../presto/exception/TestPrestoException.scala | 36 - .../factory/TestPrestoEngineConnFactory.scala | 42 - .../presto/utils/testPrestoSQLHook.scala | 32 - linkis-engineconn-plugins/python/pom.xml | 10 - .../python/PythonEngineConnPlugin.java | 80 ++ .../python/conf/PythonEngineConfiguration.java | 51 + .../python/exception/ExecuteException.java | 26 + .../python/exception/PythonExecuteError.java | 26 + .../PythonSessionStartFailedExeception.java | 27 + .../python/executor/PythonEngineConnExecutor.java | 189 ++++ .../python/hook/PythonVersionEngineHook.java | 78 ++ .../PythonProcessEngineConnLaunchBuilder.java | 22 + .../manager/engineplugin/python/utils/Kind.java | 92 ++ .../python/src/main/resources/log4j2.xml | 6 +- .../python/PythonEngineConnPlugin.scala | 75 -- .../python/conf/PythonEngineConfiguration.scala | 49 - .../exception/NoSupportEngineException.scala | 26 - .../exception/PythonSessionNullException.scala | 35 - .../python/executor/PythonEngineConnExecutor.scala | 172 --- .../python/executor/PythonSession.scala | 22 +- .../python/factory/PythonEngineConnFactory.scala | 2 +- .../python/hook/PythonVersionEngineHook.scala | 67 -- .../PythonProcessEngineConnLaunchBuilder.scala | 22 - .../manager/engineplugin/python/utils/Kind.scala | 104 -- .../manager/engineplugin/python/utils/State.scala | 74 -- .../python/TestPythonEngineConnPlugin.java | 30 + .../python/conf/TestPythonEngineConfiguration.java | 41 + .../exception/TestNoSupportEngineException.java | 47 + .../executor/TestPythonEngineConnExecutor.java | 82 ++ .../factory/TestPythonEngineConnFactory.java | 44 + .../engineplugin/python/utils/TestKind.java | 42 + .../python/TestPythonEngineConnPlugin.scala | 30 - .../conf/TestPythonEngineConfiguration.scala | 43 - .../exception/TestNoSupportEngineException.scala | 95 -- .../executor/TestPythonEngineConnExecutor.scala | 82 -- .../factory/TestPythonEngineConnFactory.scala | 42 - .../engineplugin/python/utils/TestKind.scala | 43 - .../engineplugin/python/utils/TestState.scala | 37 - linkis-engineconn-plugins/seatunnel/pom.xml | 20 +- .../seatunnel/client/LinkSeatunnelZetaClient.java | 74 ++ .../client/LinkisSeatunnelFlinkClient.java | 67 -- .../client/LinkisSeatunnelFlinkSQLClient.java | 67 -- .../client/LinkisSeatunnelFlinkV2Client.java | 72 ++ .../client/LinkisSeatunnelSparkClient.java | 74 -- .../client/LinkisSeatunnelSparkV2Client.java | 74 ++ .../errorcode/SeatunnelErrorCodeSummary.java | 2 +- .../org/apache/seatunnel/common/config/Common.java | 130 ++- .../seatunnel/core/base/config/ConfigBuilder.java | 26 +- .../seatunnel/core/base/config/PluginFactory.java | 237 ----- .../apache/seatunnel/core/flink/FlinkStarter.java | 74 -- .../seatunnel/core/flink/FlinkV2Starter.java | 99 ++ .../apache/seatunnel/core/spark/SparkStarter.java | 350 ------ .../seatunnel/core/spark/SparkV2Starter.java | 410 ++++++++ .../apache/seatunnel/core/sql/FlinkSqlStarter.java | 69 -- .../apache/seatunnel/core/zeta/ZetaStarter.java | 90 ++ .../seatunnel/src/main/resources/log4j2.xml | 6 +- .../config/SeatunnelFlinkEnvConfiguration.scala | 9 +- .../config/SeatunnelZetaEnvConfiguration.scala | 41 + .../executor/SeatunnelFlinkOnceCodeExecutor.scala | 21 +- .../SeatunnelFlinkSQLOnceCodeExecutor.scala | 184 ---- .../executor/SeatunnelSparkOnceCodeExecutor.scala | 4 +- .../executor/SeatunnelZetaOnceCodeExecutor.scala | 169 +++ .../factory/SeatunnelEngineConnFactory.scala | 11 +- .../factory/SeatunnelFlinkSQLExecutorFactory.scala | 46 - .../factory/SeatunnelZetaExecutorFactory.scala | 45 + .../seatunnel/util/SeatunnelUtils.scala | 15 +- linkis-engineconn-plugins/shell/pom.xml | 10 - .../engineplugin/shell/ShellEngineConnPlugin.java | 81 ++ .../ShellProcessEngineConnLaunchBuilder.java | 22 + .../shell/common/ShellEngineConnPluginConst.java | 24 + .../shell/conf/ShellEngineConnConf.java | 30 + .../shell/exception/NoCorrectUserException.java | 28 + .../shell/exception/ShellCodeErrorException.java | 28 + .../engineplugin/shell/executor/ReaderThread.java | 97 ++ .../shell/executor/ShellECTaskInfo.java | 54 + .../ShellEngineConnConcurrentExecutor.java | 151 +++ .../shell/executor/ShellEngineConnExecutor.java | 339 ++++++ .../shell/executor/YarnAppIdExtractor.java | 53 + .../resources/conf/linkis-engineconn.properties | 3 +- .../shell/src/main/resources/conf/log4j2.xml | 10 +- .../engineplugin/shell/ShellEngineConnPlugin.scala | 76 -- .../ShellProcessEngineConnLaunchBuilder.scala | 22 - .../shell/common/ShellEnginePluginConst.scala | 23 - .../shell/conf/ShellEngineConnConf.scala | 30 - .../shell/exception/NoCorrectUserException.scala | 27 - .../engineplugin/shell/executor/ReaderThread.scala | 96 -- .../shell/executor/ShellECTaskInfo.scala | 20 - .../ShellEngineConnConcurrentExecutor.scala | 348 ------ .../shell/executor/ShellEngineConnExecutor.scala | 319 ------ .../shell/executor/YarnAppIdExtractor.scala | 81 -- .../shell/factory/ShellEngineConnFactory.scala | 0 .../shell/TestShellEngineConnPlugin.java | 33 + .../common/TestShellEngineConnPluginConst.java | 32 + .../exception/TestNoCorrectUserException.java | 34 + .../executor/TestShellEngineConnExecutor.java | 62 ++ .../shell/TestShellEngineConnPlugin.scala | 33 - .../common/TestShellEngineConnPluginConst.scala | 33 - .../exception/TestNoCorrectUserException.scala | 38 - .../executor/TestShellEngineConnExecutor.scala | 62 -- linkis-engineconn-plugins/spark/pom.xml | 228 +--- linkis-engineconn-plugins/spark/scala-2.12/pom.xml | 970 +++++++++++++++++ .../scala-2.12/src/main/assembly/distribution.xml | 44 + .../scala-2.12/src/test/resources/etltest.dolphin | 4 + .../spark/datacalc/TestDeltaCalc.scala | 141 +++ .../spark/datacalc/TestElasticsearchCala.scala | 122 +++ .../engineplugin/spark/datacalc/TestHudiCalc.scala | 146 +++ .../spark/datacalc/TestMongoCala.scala | 120 +++ .../engineplugin/spark/datacalc/TestSolrCala.scala | 121 +++ .../spark/datacalc/TestStarrocksCala.scala | 128 +++ .../spark/client/context/SparkConfig.java | 13 +- .../YarnApplicationClusterDescriptorAdapter.java | 2 - .../datacalc/service/LinkisDataSourceContext.java | 4 + .../datacalc/service/strategy/DmStrategy.java | 36 + .../service/strategy/KingbaseStrategy.java | 36 + .../spark/datacalc/sink/DataLakeSinkConfig.java | 71 ++ .../datacalc/sink/ElasticsearchSinkConfig.java | 101 ++ .../spark/datacalc/sink/KafkaSinkConfig.java | 70 ++ .../spark/datacalc/sink/MongoSinkConfig.java | 71 ++ .../spark/datacalc/sink/RedisSinkConfig.java | 90 ++ .../spark/datacalc/sink/RocketmqSinkConfig.java | 66 ++ .../spark/datacalc/sink/SolrSinkConfig.java | 81 ++ .../spark/datacalc/sink/StarrocksSinkConfig.java | 85 ++ .../datacalc/source/DataLakeSourceConfig.java | 56 + .../datacalc/source/ElasticsearchSourceConfig.java | 95 ++ .../spark/datacalc/source/KafkaSourceConfig.java | 55 + .../spark/datacalc/source/MongoSourceConfig.java | 55 + .../spark/datacalc/source/RedisSourceConfig.java | 94 ++ .../datacalc/source/RocketmqSourceConfig.java | 55 + .../spark/datacalc/source/SolrSourceConfig.java | 45 + .../datacalc/source/StarrocksSourceConfig.java | 73 ++ .../spark/datacalc/util/PluginUtil.java | 16 + .../spark/src/main/resources/log4j2.xml | 14 +- .../spark/Interpreter/Interpreter.scala | 43 - .../spark/Interpreter/ProcessInterpreter.scala | 125 --- .../spark/Interpreter/PythonInterpreter.scala | 155 +-- .../engineplugin/spark/SparkEngineConnPlugin.scala | 4 - .../engineplugin/spark/common/SparkKind.scala | 23 - .../spark/config/SparkConfiguration.scala | 2 + .../spark/datacalc/DataCalcExecution.scala | 10 +- .../spark/datacalc/sink/DataLakeSink.scala | 37 + .../spark/datacalc/sink/ElasticsearchSink.scala | 55 + .../spark/datacalc/sink/FileSink.scala | 1 + .../spark/datacalc/sink/KafkaSink.scala | 47 + .../spark/datacalc/sink/MongoSink.scala | 52 + .../spark/datacalc/sink/RedisSink.scala | 53 + .../spark/datacalc/sink/RocketmqSink.scala | 60 ++ .../spark/datacalc/sink/SolrSink.scala | 52 + .../spark/datacalc/sink/StarrocksSink.scala | 54 + .../spark/datacalc/source/DataLakeSource.scala | 41 + .../datacalc/source/ElasticsearchSource.scala | 48 + .../spark/datacalc/source/FileSource.scala | 1 + .../spark/datacalc/source/KafkaSource.scala | 42 + .../spark/datacalc/source/MongoSource.scala | 45 + .../spark/datacalc/source/RedisSource.scala | 50 + .../spark/datacalc/source/RocketmqSource.scala | 44 + .../spark/datacalc/source/SolrSource.scala | 44 + .../spark/datacalc/source/StarrocksSource.scala | 49 + .../engineplugin/spark/executor/SQLSession.scala | 6 +- .../spark/executor/SparkEngineConnExecutor.scala | 12 +- .../spark/executor/SparkPythonExecutor.scala | 4 +- .../spark/executor/SparkScalaExecutor.scala | 4 +- .../spark/factory/SparkEngineConnFactory.scala | 4 + .../engineplugin/spark/imexport/ExportData.scala | 13 +- .../engineplugin/spark/imexport/LoadData.scala | 33 +- ...SparkSubmitProcessEngineConnLaunchBuilder.scala | 2 +- .../engineplugin/spark/utils/EngineUtils.scala | 4 +- .../execution/datasources/csv/DolphinToSpark.scala | 26 +- .../spark/src/test/resources/etltest.dolphin | 4 + .../spark/datacalc/TestExcelCala.scala | 135 +++ .../spark/datacalc/TestKafkaCala.scala | 118 +++ .../spark/datacalc/TestRedisCalc.scala | 200 ++++ .../spark/datacalc/TestRocketmqCala.scala | 71 ++ linkis-engineconn-plugins/sqoop/pom.xml | 20 +- .../sqoop/src/main/resources/log4j2.xml | 10 +- linkis-engineconn-plugins/trino/pom.xml | 4 +- .../engineplugin/trino/TrinoEngineConnPlugin.java | 78 ++ .../TrinoProcessEngineConnLaunchBuilder.java | 35 + .../trino/conf/TrinoConfiguration.java | 90 ++ .../engineplugin/trino/conf/TrinoEngineConfig.java | 53 + .../trino/exception/TrinoClientException.java | 26 + .../trino/exception/TrinoGrantmaException.java | 26 + .../exception/TrinoModifySchemaException.java | 26 + .../exception/TrinoStateInvalidException.java | 26 + .../trino/executor/TrinoEngineConnExecutor.java | 561 ++++++++++ .../trino/password/CommandPasswordCallback.java | 75 -- .../trino/password/StaticPasswordCallback.java | 40 - .../linkis/engineplugin/trino/utils/TrinoCode.java | 47 + .../engineplugin/trino/utils/TrinoSQLHook.java | 34 + .../main/resources/linkis-engineconn.properties | 1 + .../trino/src/main/resources/log4j2.xml | 6 +- .../engineplugin/trino/TrinoEngineConnPlugin.scala | 66 -- .../TrinoProcessEngineConnLaunchBuilder.scala | 37 - .../trino/conf/TrinoConfiguration.scala | 71 -- .../trino/conf/TrinoEngineConfig.scala | 48 - .../trino/exception/TrinoException.scala | 32 - .../trino/executor/TrinoEngineConnExecutor.scala | 534 ---------- .../engineplugin/trino/utils/TrinoCode.scala | 53 - .../engineplugin/trino/utils/TrinoSQLHook.scala | 36 - .../linkis/storage/io/client/DefaultIOClient.scala | 4 +- .../io/iteraceptor/IOMethodInterceptor.scala | 14 +- .../IOMethodInterceptorCreatorImpl.scala | 4 +- .../linkis/storage/io/utils/IOClientUtils.scala | 3 +- linkis-hadoop-hdfs-client-shade/pom.xml | 3 +- .../catalyst/planner/TaskPlannerTransform.scala | 28 +- .../physical/CodeLogicalUnitExecTask.scala | 1 + .../service/ComputationTaskExecutionReceiver.scala | 23 +- .../ecm/ComputationEngineConnManager.scala | 30 +- .../ecm/cache/EngineAsyncResponseCache.scala | 16 +- .../impl/ComputationEngineConnExecutor.scala | 2 +- .../impl/DefaultEngineAsyncResponseService.scala | 12 +- .../strategy/async/AsyncExecTaskRunnerImpl.scala | 50 +- .../utils/OrchestratorLoggerUtils.scala | 37 + linkis-orchestrator/pom.xml | 1 + .../server/response/EngineLabelResponse.java | 2 +- .../mapper/common/CgManagerLabelMapper.xml | 6 +- .../mapper/common/ConfigurationConfigKeyMapper.xml | 41 - .../common/ConfigurationConfigValueMapper.xml | 11 +- .../ConfigurationKeyEngineRelationMapper.xml | 5 +- .../mapper/common/DatasourceAccessMapper.xml | 8 +- .../mapper/common/DatasourceEnvMapper.xml | 10 +- .../mapper/common/DatasourceTypeKeyMapper.xml | 64 -- .../mapper/common/DatasourceTypeMapper.xml | 50 - .../common/EngineConnPluginBmlResourcesMapper.xml | 98 +- .../mapper/common/GatewayAuthTokenMapper.xml | 10 +- .../resources/mapper/common/PsErrorCodeMapper.xml | 47 - .../common/RmExternalResourceProviderMapper.xml | 12 +- .../resources/mapper/common/UdfManagerMapper.xml | 6 +- .../main/resources/mapper/common/UdfTreeMapper.xml | 12 +- .../mapper/mysql/ConfigurationConfigKeyMapper.xml | 31 + .../mapper/mysql/DatasourceTypeKeyMapper.xml | 64 ++ .../mapper/mysql/DatasourceTypeMapper.xml | 50 + .../resources/mapper/mysql/PsErrorCodeMapper.xml | 47 + .../postgresql/ConfigurationConfigKeyMapper.xml | 68 ++ .../mapper/postgresql/DatasourceTypeKeyMapper.xml | 64 ++ .../mapper/postgresql/DatasourceTypeMapper.xml | 50 + .../mapper/postgresql/PsErrorCodeMapper.xml | 47 + .../server/dao/CgManagerLabelMapperTest.java | 45 + .../dao/ConfigurationConfigKeyMapperTest.java | 86 ++ .../dao/ConfigurationConfigValueMapperTest.java | 65 ++ .../ConfigurationKeyEngineRelationMapperTest.java | 53 + .../server/dao/DatasourceEnvMapperTest.java | 55 + .../server/dao/DatasourceTypeKeyMapperTest.java | 39 + .../server/dao/DatasourceTypeMapperTest.java | 39 + .../EngineConnPluginBmlResourcesMapperTest.java | 56 + .../server/dao/GatewayAuthTokenMapperTest.java | 55 + .../server/dao/PsErrorCodeMapperTest.java | 50 + .../dao/RmExternalResourceProviderMapperTest.java | 52 + .../server/dao/UdfManagerMapperTest.java | 47 + .../server/dao/UdfTreeMapperTest.java | 54 + .../src/test/resources/application.properties | 17 +- .../src/test/resources/create.sql | 202 +++- .../src/test/resources/create_pg.sql | 252 +++++ .../src/test/resources/data.sql | 216 ---- .../linkis-bml/linkis-bml-server/pom.xml | 4 - .../linkis/bml/service/impl/TaskServiceImpl.java | 2 +- .../resources/mapper/common/BmlProjectMapper.xml | 62 -- .../resources/mapper/common/DownloadMapper.xml | 6 +- .../resources/mapper/common/ResourceMapper.xml | 107 -- .../main/resources/mapper/common/TaskMapper.xml | 4 +- .../main/resources/mapper/common/VersionMapper.xml | 194 ---- .../resources/mapper/mysql/BmlProjectMapper.xml | 78 ++ .../main/resources/mapper/mysql/ResourceMapper.xml | 121 +++ .../main/resources/mapper/mysql/VersionMapper.xml | 243 +++++ .../mapper/postgresql/BmlProjectMapper.xml | 81 ++ .../resources/mapper/postgresql/ResourceMapper.xml | 110 ++ .../resources/mapper/postgresql/VersionMapper.xml | 187 ++++ .../apache/linkis/bml/dao/BmlProjectDaoTest.java | 5 + .../src/test/resources/application.properties | 4 + .../src/test/resources/create_pg.sql | 129 +++ .../LinkisConfigurationErrorCodeSummary.java | 2 +- .../restful/api/ConfigurationRestfulApi.java | 36 +- .../restful/api/TenantConfigrationRestfulApi.java | 13 +- .../configuration/service/TenantConfigService.java | 2 +- .../service/impl/TenantConfigServiceImpl.java | 73 +- .../main/resources/mapper/common/LabelMapper.xml | 12 +- .../main/resources/mapper/common/UserIpMapper.xml | 91 -- .../resources/mapper/common/UserTenantMapper.xml | 92 -- .../mapper/{common => mysql}/ConfigMapper.xml | 0 .../main/resources/mapper/mysql/UserIpMapper.xml | 91 ++ .../resources/mapper/mysql/UserTenantMapper.xml | 92 ++ .../resources/mapper/postgresql/ConfigMapper.xml | 265 +++++ .../resources/mapper/postgresql/UserIpMapper.xml | 91 ++ .../mapper/postgresql/UserTenantMapper.xml | 92 ++ .../linkis/configuration/conf/Configuration.scala | 3 + .../linkis/configuration/dao/ConfigMapperTest.java | 5 + .../linkis/configuration/dao/UserIpMapperTest.java | 91 ++ .../configuration/dao/UserTenantMapperTest.java | 84 ++ .../restful/api/ConfigurationRestfulApiTest.java | 8 +- .../src/test/resources/application.properties | 16 +- .../src/test/resources/create.sql | 65 +- .../src/test/resources/create_pg.sql | 148 +++ .../src/test/resources/data.sql | 51 - .../client/builder/RPCHttpContextClientConfig.java | 20 - .../listener/callback/AbstractCallbackEngine.java | 24 - .../mapper/common/contextHistoryMapper.xml | 79 -- .../mapper/common/contextIDListenerMapper.xml | 46 - .../resources/mapper/common/contextIDMapper.xml | 105 -- .../mapper/common/contextKeyListenerMapper.xml | 4 +- .../resources/mapper/common/contextMapMapper.xml | 168 --- .../mapper/mysql/contextHistoryMapper.xml | 89 ++ .../mapper/mysql/contextIDListenerMapper.xml | 51 + .../resources/mapper/mysql/contextIDMapper.xml | 105 ++ .../resources/mapper/mysql/contextMapMapper.xml | 168 +++ .../mapper/postgresql/contextHistoryMapper.xml | 83 ++ .../mapper/postgresql/contextIDListenerMapper.xml | 50 + .../mapper/postgresql/contextIDMapper.xml | 110 ++ .../mapper/postgresql/contextMapMapper.xml | 172 +++ .../persistence/dao/ContextHistoryMapperTest.java | 6 +- .../cs/persistence/dao/ContextIDMapperTest.java | 12 +- .../cs/persistence/dao/ContextMapMapperTest.java | 266 +++++ .../src/test/resources/application.properties | 3 + .../linkis-cs-server/src/test/resources/create.sql | 16 + .../src/test/resources/create_pg.sql | 90 ++ .../client/request/MetadataGetColumnsAction.scala | 5 + .../request/MetadataGetDatabasesAction.scala | 3 + .../client/request/MetadataGetTablesAction.scala | 4 + .../GetConnectParamsByDataSourceIdResult.scala | 2 +- .../response/GetInfoByDataSourceIdResult.scala | 2 +- .../response/MetadataGetTablePropsResult.scala | 2 +- .../core/restful/DataSourceCoreRestfulApi.java | 2 +- .../resources/mapper/common/DataSouceMapper.xml | 144 +-- .../mapper/common/DataSourceEnvMapper.xml | 58 +- .../mapper/common/DataSourceParamKeyMapper.xml | 60 -- .../mapper/common/DataSourceTypeMapper.xml | 53 - .../mapper/common/DataSourceVersionMapper.xml | 70 -- .../mapper/mysql/DataSourceParamKeyMapper.xml | 60 ++ .../mapper/mysql/DataSourceTypeMapper.xml | 53 + .../mapper/mysql/DataSourceVersionMapper.xml | 70 ++ .../mapper/postgresql/DataSourceParamKeyMapper.xml | 59 ++ .../mapper/postgresql/DataSourceTypeMapper.xml | 52 + .../mapper/postgresql/DataSourceVersionMapper.xml | 71 ++ .../src/test/resources/application.properties | 14 +- .../server/src/test/resources/create_pg.sql | 119 +++ .../query/common/cache/CacheConfiguration.java | 7 +- .../query/common/domain/DataSourceTypeEnum.java | 34 + .../query/common/domain/GenerateSqlInfo.java | 57 + .../query/common/domain/MetaColumnInfo.java | 9 + .../common/service/AbstractDbMetaService.java | 34 + .../query/common/service/GenerateSqlTemplate.java | 68 ++ .../query/common/service/MetadataDbService.java | 20 + .../server/loader/MetaClassLoaderManager.java | 34 +- .../query/server/restful/MetadataQueryRestful.java | 114 ++ .../query/server/service/MetadataQueryService.java | 39 + .../service/impl/MetadataQueryServiceImpl.java | 205 ++++ .../linkis-metadata-query/service/hive/pom.xml | 6 +- .../metadata/query/service/HiveConnection.java | 10 - .../metadata/query/service/HiveMetaService.java | 1 - .../linkis-metadata-query/service/jdbc/pom.xml | 5 - .../query/service/AbstractSqlConnection.java | 237 +++++ .../query/service/ClickhouseMetaService.java | 24 +- .../metadata/query/service/Db2MetaService.java | 24 +- .../metadata/query/service/DmMetaService.java | 24 +- .../metadata/query/service/DorisMetaService.java | 94 -- .../query/service/GreenplumMetaService.java | 23 +- .../query/service/KingbaseMetaService.java | 24 +- .../metadata/query/service/MysqlMetaService.java | 16 + .../metadata/query/service/OracleMetaService.java | 24 +- .../query/service/PostgresqlMetaService.java | 23 +- .../query/service/SqlserverMetaService.java | 24 +- .../query/service/clickhouse/SqlConnection.java | 116 +- .../metadata/query/service/db2/SqlConnection.java | 155 +-- .../metadata/query/service/dm/SqlConnection.java | 113 +- .../query/service/greenplum/SqlConnection.java | 144 +-- .../query/service/kingbase/SqlConnection.java | 133 +-- .../query/service/mysql/SqlConnection.java | 133 +-- .../query/service/oracle/SqlConnection.java | 128 +-- .../query/service/postgres/SqlConnection.java | 144 +-- .../query/service/sqlserver/SqlConnection.java | 119 +-- .../metadata/query/service/KafkaMetaService.java | 5 - .../linkis-metadata-query/service/mongodb/pom.xml | 2 +- .../main/resources/mapper/common/HiveMetaDao.xml | 204 ---- .../src/main/resources/mapper/common/MdqDao.xml | 93 -- .../main/resources/mapper/mysql/HiveMetaDao.xml | 273 +++++ .../src/main/resources/mapper/mysql/MdqDao.xml | 100 ++ .../resources/mapper/postgresql/HiveMetaDao.xml | 221 ++++ .../main/resources/mapper/postgresql/MdqDao.xml | 91 ++ .../linkis/metadata/ddl/ImportDDLCreator.scala | 1 - .../src/test/resources/application.properties | 4 + .../src/test/resources/create_pg.sql | 233 ++++ .../src/test/resources/application.properties | 15 +- .../src/test/resources/create.sql | 6 +- .../src/test/resources/create_pg.sql | 32 + .../src/test/resources/data.sql | 20 - .../mapper/common/InsLabelRelationMapper.xml | 203 ---- .../resources/mapper/common/InstanceInfoMapper.xml | 53 - .../mapper/common/InstanceLabelMapper.xml | 115 -- .../mapper/mysql/InsLabelRelationMapper.xml | 203 ++++ .../resources/mapper/mysql/InstanceInfoMapper.xml | 55 + .../resources/mapper/mysql/InstanceLabelMapper.xml | 115 ++ .../mapper/postgresql/InsLabelRelationMapper.xml | 202 ++++ .../mapper/postgresql/InstanceInfoMapper.xml | 55 + .../mapper/postgresql/InstanceLabelMapper.xml | 114 ++ .../label/dao/InsLabelRelationDaoTest.java | 10 + .../instance/label/dao/InstanceInfoDaoTest.java | 5 + .../instance/label/dao/InstanceLabelDaoTest.java | 5 + .../src/test/resources/application.properties | 3 + .../src/test/resources/create_pg.sql | 65 ++ .../errorcode/JobhistoryErrorCodeSummary.java | 2 +- .../resources/mapper/common/JobDetailMapper.xml | 22 +- .../resources/mapper/common/JobHistoryMapper.xml | 232 ---- .../resources/mapper/mysql/JobHistoryMapper.xml | 234 +++++ .../mapper/postgresql/JobHistoryMapper.xml | 232 ++++ .../service/impl/JobHistoryQueryServiceImpl.scala | 2 + .../linkis/jobhistory/dao/JobDetailMapperTest.java | 95 ++ .../restful/api/QueryRestfulApiTest.java | 4 +- .../src/test/resources/application.properties | 4 + .../src/test/resources/create_pg.sql | 62 ++ .../resources/mapper/common/CommonLockMapper.xml | 10 +- .../src/test/resources/application.properties | 15 +- .../src/test/resources/create.sql | 5 + .../src/test/resources/create_pg.sql | 31 + .../src/test/resources/data.sql | 22 - .../filesystem/restful/api/BMLFsRestfulApi.java | 11 +- .../filesystem/restful/api/FsRestfulApi.java | 36 +- .../src/test/resources/application.properties | 2 +- .../udf/service/impl/UDFTreeServiceImpl.java | 2 +- .../src/main/resources/mapper/common/UDFDao.xml | 397 ------- .../main/resources/mapper/common/UDFTreeDao.xml | 56 +- .../main/resources/mapper/common/UDFVersionDao.xml | 114 -- .../src/main/resources/mapper/mysql/UDFDao.xml | 434 ++++++++ .../main/resources/mapper/mysql/UDFVersionDao.xml | 130 +++ .../main/resources/mapper/postgresql/UDFDao.xml | 404 +++++++ .../resources/mapper/postgresql/UDFVersionDao.xml | 119 +++ .../src/test/resources/application.properties | 17 +- .../src/test/resources/create.sql | 47 +- .../src/test/resources/create_pg.sql | 143 +++ .../linkis-udf-service/src/test/resources/data.sql | 61 -- .../src/main/resources/mapper/common/VarMapper.xml | 60 -- .../src/main/resources/mapper/mysql/VarMapper.xml | 64 ++ .../main/resources/mapper/postgresql/VarMapper.xml | 61 ++ .../src/test/resources/application.properties | 16 +- .../linkis-variable/src/test/resources/create.sql | 18 +- .../src/test/resources/create_pg.sql | 59 ++ .../linkis-variable/src/test/resources/data.sql | 32 - .../linkis-service-discovery/linkis-eureka/pom.xml | 51 +- .../src/main/assembly/distribution.xml | 14 - .../main/resources/mapper/common/TokenMapper.xml | 6 +- .../src/test/resources/application.properties | 17 +- .../src/test/resources/create.sql | 14 +- .../src/test/resources/create_pg.sql | 45 + .../src/test/resources/data.sql | 28 - .../linkis/gateway/security/GatewaySSOUtils.scala | 12 +- .../linkis/gateway/security/UserRestful.scala | 10 +- .../linkis/httpclient/dws/response/DWSResult.scala | 4 +- .../linkis-gateway-server-support/pom.xml | 11 +- linkis-web/.env | 1 + linkis-web/package.json | 34 +- .../release-docs/licenses/LICENSE-autoprefixer.txt | 20 + .../release-docs/licenses/LICENSE-postcss.txt | 20 + .../apps/URM/module/functionManagement/index.vue | 2 +- .../src/apps/URM/module/udfManagement/index.vue | 2 +- .../src/apps/linkis/assets/styles/console.scss | 7 +- .../src/apps/linkis/components/variable/index.scss | 1 + linkis-web/src/apps/linkis/i18n/common/en.json | 15 +- linkis-web/src/apps/linkis/i18n/common/zh.json | 15 +- .../src/apps/linkis/module/ECM/engineConn.vue | 5 +- linkis-web/src/apps/linkis/module/ECM/index.vue | 2 +- linkis-web/src/apps/linkis/module/ECM/log.vue | 6 +- .../apps/linkis/module/datasourceAccess/index.vue | 4 +- .../linkis/module/datasourceEnv/EditForm/index.vue | 17 +- .../src/apps/linkis/module/datasourceEnv/index.vue | 24 +- .../apps/linkis/module/datasourceType/index.vue | 4 +- .../apps/linkis/module/datasourceTypeKey/index.vue | 4 +- .../src/apps/linkis/module/errorCode/index.vue | 5 +- .../apps/linkis/module/gatewayAuthToken/index.vue | 6 +- .../module/globalHistoryManagement/index.vue | 2 +- .../module/globalHistoryManagement/viewHistory.vue | 1 + .../apps/linkis/module/ipListManagement/index.vue | 2 +- .../linkis/module/microServiceManagement/index.vue | 2 +- .../module/resourceManagement/engineConn.vue | 3 +- .../linkis/module/resourceManagement/index.vue | 8 +- .../apps/linkis/module/resourceManagement/log.vue | 6 +- .../module/rmExternalResourceProvider/index.vue | 13 +- .../src/apps/linkis/module/setting/setting.vue | 9 +- .../linkis/module/tenantTagManagement/index.vue | 25 +- .../src/apps/linkis/module/udfManager/index.vue | 5 +- .../apps/linkis/module/udfTree/EditForm/index.vue | 109 +- .../src/apps/linkis/module/udfTree/index.vue | 38 +- .../src/apps/linkis/module/udfTree/service.js | 4 +- linkis-web/src/apps/linkis/view/linkis/index.vue | 2 +- linkis-web/src/common/i18n/en.json | 1 + linkis-web/src/common/i18n/zh.json | 1 + .../components/consoleComponent/resultSetList.vue | 6 +- .../src/components/consoleComponent/toolbar.vue | 7 + pom.xml | 675 +++++++++--- tool/dependencies/known-dependencies.txt | 351 ++++--- 1476 files changed, 65746 insertions(+), 38817 deletions(-) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
