This is an automated email from the ASF dual-hosted git repository. peacewong pushed a commit to branch dev-1.1.3 in repository https://gitbox.apache.org/repos/asf/incubator-linkis.git
commit a2ffc2ceef013cd8a779d1de5c2c6096f53588bb Merge: b20182d28 4963d6232 Author: peacewong <[email protected]> AuthorDate: Fri May 20 16:02:09 2022 +0800 Merge pull request #2139 from apache/master Merge master into dev-1.1.3 .mvn/wrapper/maven-wrapper.jar | Bin 50710 -> 0 bytes DISCLAIMER | 6 + DISCLAIMER-WIP | 17 - LICENSE | 15 +- LICENSE-binary | 3 + LICENSE-binary-ui | 4 +- NOTICE-binary | 24 +- .../conf/linkis-ps-publicservice.properties | 2 + .../assembly-combined/conf/linkis.properties | 2 +- .../assembly-combined/pom.xml | 2 +- .../public-module-combined/pom.xml | 17 +- .../assembly-combined/sbin/common.sh | 2 +- assembly-combined-package/bin/checkEnv.sh | 20 +- assembly-combined-package/bin/common.sh | 2 +- assembly-combined-package/bin/install.sh | 85 ++-- .../deploy-config/linkis-env.sh | 11 +- assembly-combined-package/pom.xml | 2 +- db/linkis_ddl.sql | 2 +- db/linkis_dml.sql | 25 +- db/upgrade/1.1.1_schema/mysql/linkis_ddl.sql | 117 +++++ db/upgrade/1.1.1_schema/mysql/linkis_dml.sql | 197 ++++++++ LICENSE => licenses-binary/LICENSE-sqoop.txt | 168 ++++++- LICENSE => licenses-binary/LICENSE-tephra.txt | 62 --- LICENSE => licenses-binary/LICENSE-twill.txt | 37 +- .../LICENSE-py4j-0.10.9.5-src.txt | 2 +- linkis-commons/linkis-common/pom.xml | 4 +- .../linkis/common/utils/CodeAndRunTypeUtils.scala | 77 +++ .../common/utils/CodeAndRunTypeUtilsTest.scala | 63 +++ .../linkis/common/utils/OverloadUtilsTest.scala | 35 -- linkis-commons/linkis-hadoop-common/pom.xml | 2 +- linkis-commons/linkis-httpclient/pom.xml | 2 +- linkis-commons/linkis-module/pom.xml | 7 +- linkis-commons/linkis-mybatis/pom.xml | 2 +- linkis-commons/linkis-protocol/pom.xml | 2 +- linkis-commons/linkis-rpc/pom.xml | 11 +- .../linkis/rpc/message/method/MessageExecutor.java | 11 +- .../message/utils/LoadBalancerOptionsUtils.java | 43 ++ .../apache/linkis/rpc/conf/RPCConfiguration.scala | 2 + .../linkis/rpc/sender/SpringMVCRPCSender.scala | 6 +- linkis-commons/linkis-scheduler/pom.xml | 2 +- linkis-commons/linkis-storage/pom.xml | 20 +- .../storage/factory/impl/BuildHDFSFileSystem.java | 2 +- .../storage/factory/impl/BuildLocalFileSystem.java | 2 +- .../apache/linkis/storage/csv/CSVFsWriter.scala | 3 +- .../linkis/storage/csv/StorageCSVWriter.scala | 16 +- .../storage/io/IOMethodInterceptorCreator.scala | 3 +- .../linkis/storage/script/ScriptFsWriter.scala | 6 +- .../linkis/storage/script/VariableParser.scala | 2 +- .../script/compaction/PYScriptCompaction.scala | 9 +- .../script/compaction/QLScriptCompaction.scala | 10 +- .../script/compaction/ScalaScriptCompaction.scala | 7 +- .../script/compaction/ShellScriptCompaction.scala | 11 +- .../storage/script/parser/PYScriptParser.scala | 8 +- .../storage/script/parser/QLScriptParser.scala | 10 +- .../storage/script/parser/ScalaScriptParser.scala | 9 +- ...YScriptParser.scala => ShellScriptParser.scala} | 23 +- linkis-commons/pom.xml | 2 +- .../linkis-cli/linkis-cli-application/pom.xml | 2 +- .../linkis-cli/linkis-cli-common/pom.xml | 2 +- .../linkis-cli/linkis-cli-core/pom.xml | 2 +- .../handler/CommandExceptionHandlerTest.java | 60 ++- .../command/fitter/SingleTplFitterTest.java | 3 +- .../command/parser/SingleCmdParserTest.java | 45 +- .../interactor/properties/StdPropsLoaderTest.java | 24 +- .../linkis-client/linkis-cli/pom.xml | 2 +- .../linkis-computation-client/pom.xml | 2 +- .../linkis-computation-governance-common/pom.xml | 2 +- .../governance/common/paser/CodeParser.scala | 20 +- .../linkis/governance/common/utils/JobUtils.scala | 42 +- .../governance/common/utils/JobUtilsTest.scala | 45 ++ .../linkis-engineconn-linux-launch/pom.xml | 2 +- .../linkis-engineconn-manager-core/pom.xml | 2 +- .../core/launch/ProcessEngineCommandBuilder.scala | 2 +- .../linkis-engineconn-manager-server/pom.xml | 2 +- .../impl/AbstractEngineConnLaunchService.scala | 11 +- .../linkis-engineconn-manager/pom.xml | 2 +- .../linkis-once-engineconn/pom.xml | 2 +- .../linkis-streaming-engineconn/pom.xml | 2 +- .../linkis-computation-engineconn/pom.xml | 2 +- .../executor/async/AsyncExecuteContext.scala | 6 + .../executor/conf/ComputationExecutorConf.scala | 4 + .../executor/execute/ComputationExecutor.scala | 4 +- .../hook/executor/FunctionExecutorHook.scala | 13 +- .../service/TaskExecutionServiceImpl.scala | 14 +- .../computation/executor/utlis/ProgressUtils.scala | 36 +- .../linkis-engineconn-common/pom.xml | 2 +- .../linkis-engineconn-core/pom.xml | 2 +- .../accessible-executor/pom.xml | 2 +- .../conf/AccessibleExecutorConfiguration.scala | 1 + .../execution/AccessibleEngineConnExecution.scala | 6 +- .../callback-service/pom.xml | 2 +- .../executor-core/pom.xml | 2 +- .../resource-executor/pom.xml | 2 +- .../linkis-engineconn/pom.xml | 2 +- .../linkis-entrance/pom.xml | 2 +- .../linkis/entrance/job/EntranceExecutionJob.java | 11 + .../apache/linkis/entrance/EntranceServer.scala | 2 +- .../linkis/entrance/execute/EntranceJob.scala | 15 +- .../entrance/interceptor/RuntypeInterceptor.scala | 13 +- .../interceptor/impl/CommentInterceptor.scala | 12 +- .../interceptor/impl/CustomVariableUtils.scala | 30 +- .../impl/PythonCodeCheckInterceptor.scala | 7 +- .../interceptor/impl/SQLCodeCheckInterceptor.scala | 6 +- .../impl/SQLLimitEntranceInterceptor.scala | 6 +- .../interceptor/impl/ScalaCodeInterceptor.scala | 9 +- .../impl/SparkCodeCheckInterceptor.scala | 6 +- .../entrance/timeout/JobTimeoutManager.scala | 1 - .../linkis-jdbc-driver/pom.xml | 2 +- .../linkis-manager/label-common/pom.xml | 2 +- .../label/utils/EngineTypeLabelCreator.java | 4 + .../linkis-manager/label-manager/pom.xml | 2 +- .../linkis-application-manager/pom.xml | 2 +- .../pointer/DefaultEngineConnPluginPointer.scala | 2 + .../engine/DefaultEngineAskEngineService.scala | 18 +- .../engine/DefaultEngineCreateService.scala | 15 +- .../linkis-manager-common/pom.xml | 2 +- .../entity/persistence/PersistenceLabel.java | 2 + .../linkis-manager-service-common/pom.xml | 2 +- .../linkis-manager-persistence/pom.xml | 2 +- .../linkis/manager/dao/LabelManagerMapper.java | 5 + .../linkis/manager/dao/impl/LabelManagerMapper.xml | 4 + .../impl/DefaultLabelManagerPersistence.java | 24 +- .../linkis-manager/linkis-resource-manager/pom.xml | 2 +- .../linkis-manager/pom.xml | 2 +- linkis-computation-governance/pom.xml | 2 +- .../engineconn-plugins/flink/pom.xml | 42 +- .../flink/src/main/assembly/distribution.xml | 147 ++++++ .../deployment/ClusterDescriptorAdapter.java | 21 +- .../YarnApplicationClusterDescriptorAdapter.java | 2 +- .../flink/client/result/ChangelogResult.java | 2 +- .../flink/config/FlinkEnvConfiguration.scala | 11 + .../flink/config/FlinkResourceConfiguration.scala | 6 +- .../flink/context/EnvironmentContext.scala | 2 +- .../flink/executor/FlinkCodeOnceExecutor.scala | 2 +- .../flink/executor/FlinkJarOnceExecutor.scala | 2 +- .../flink/executor/FlinkOnceExecutor.scala | 17 +- .../flink/factory/FlinkEngineConnFactory.scala | 94 ++-- .../launch/FlinkEngineConnLaunchBuilder.scala | 2 +- .../flink/operator/TriggerSavepointOperator.scala | 43 ++ .../resource/FlinkEngineConnResourceFactory.scala | 6 +- .../engineconn-plugins/hive/pom.xml | 6 +- .../hive/executor/HiveEngineConnExecutor.scala | 17 +- .../engineconn-plugins/io_file/pom.xml | 2 +- .../engineconn-plugins/jdbc/pom.xml | 4 +- .../engineplugin/jdbc/conf/JDBCConfiguration.scala | 2 + .../jdbc/executer/JDBCEngineConnExecutor.scala | 8 +- .../engineconn-plugins/openlookeng/pom.xml | 4 +- .../openlookeng/src/main/assembly/distribution.xml | 2 +- .../executor/OpenLooKengEngineConnExecutor.java | 3 +- .../{log4j2-engineconn.xml => log4j2.xml} | 0 .../engineconn-plugins/pipeline/pom.xml | 2 +- .../conf/PipelineEngineConfiguration.scala | 4 +- .../pipeline/constant/PipeLineConstant.scala | 1 + .../pipeline/executor/CSVExecutor.scala | 5 +- .../engineconn-plugins/python/pom.xml | 2 +- .../python/src/main/py4j/py4j-0.10.7-src.zip | Bin 42437 -> 0 bytes .../python/src/main/py4j/py4j-0.10.9.5-src.zip | Bin 0 -> 42404 bytes .../python/src/main/resources/python/python.py | 1 - .../python/conf/PythonEngineConfiguration.scala | 2 +- .../engineconn-plugins/shell/pom.xml | 2 +- .../engineconn-plugins/spark/pom.xml | 20 +- .../spark/config/SparkConfiguration.scala | 2 +- .../spark/executor/SparkEngineConnExecutor.scala | 36 +- .../spark/executor/SparkScalaExecutor.scala | 2 +- .../engineconn-plugins/{hive => sqoop}/pom.xml | 246 +++------ .../sqoop/src/main/assembly/distribution.xml | 324 ++++++++++++ .../sqoop/client/LinkisSqoopClient.java | 229 +++++++++ .../engineconnplugin/sqoop/client/Sqoop.java | 551 +++++++++++++++++++++ .../sqoop/client/config/ExecutionContext.java | 12 +- .../sqoop/client/config/ParamsMapping.java | 144 ++++++ .../client/exception/JobClosableException.java | 20 +- .../client/exception/JobExecutionException.java | 19 +- .../sqoop/client/utils/JarLoader.java | 170 +++++++ .../java/org/apache/sqoop/mapreduce/JobBase.java | 410 +++++++++++++++ .../main/resources/linkis-engineconn.properties | 12 +- .../src/main/resources/log4j2.xml} | 9 +- .../sqoop/SqoopEngineConnPlugin.scala | 66 +++ .../sqoop/context/SqoopEngineConnContext.scala | 12 +- .../sqoop/context/SqoopEnvConfiguration.scala | 39 ++ .../sqoop/context/SqoopParamsConfiguration.scala | 42 ++ .../sqoop/context/SqoopResourceConfiguration.scala | 16 +- .../sqoop/executor/SqoopExecutor.scala | 50 ++ .../sqoop/executor/SqoopOnceCodeExecutor.scala | 142 ++++++ .../sqoop/executor/SqoopOnceExecutor.scala | 67 +++ .../sqoop/factory/SqoopEngineConnFactory.scala | 45 ++ .../sqoop/factory/SqoopExecutorFactory.scala | 43 ++ .../launch/SqoopEngineConnLaunchBuilder.scala | 92 ++++ .../sqoop/params/SqoopParamsResolver.scala | 21 +- .../resource/SqoopEngineConnResourceFactory.scala | 18 +- .../engineconnplugin/sqoop/util/ClassUtil.scala | 40 ++ .../linkis-engineconn-plugin-cache/pom.xml | 2 +- .../linkis-engineconn-plugin-core/pom.xml | 2 +- .../linkis-engineconn-plugin-loader/pom.xml | 2 +- .../linkis-engineconn-plugin-server/pom.xml | 2 +- linkis-engineconn-plugins/pom.xml | 3 +- linkis-extensions/linkis-io-file-client/pom.xml | 2 +- .../io/iteraceptor/IOMethodInterceptor.scala | 22 +- .../IOMethodInterceptorCreatorImpl.scala | 6 +- linkis-extensions/pom.xml | 2 +- .../linkis-code-orchestrator/pom.xml | 2 +- .../linkis-computation-orchestrator/pom.xml | 2 +- .../reheater/PruneTaskReheaterTransform.scala | 3 + .../physical/CodeLogicalUnitExecTask.scala | 3 +- .../service/ComputationTaskExecutionReceiver.scala | 2 +- .../linkis-orchestrator-core/pom.xml | 2 +- .../conf/OrchestratorConfiguration.scala | 6 +- .../orchestrator/converter/AbstractConverter.scala | 2 +- .../orchestrator/execution/TaskManager.scala | 2 + .../execution/impl/DefaultTaskManager.scala | 65 ++- .../execution/impl/NotifyTaskConsumer.scala | 21 +- .../orchestrator/listener/task/TaskInfoEvent.scala | 2 +- .../apache/linkis/orchestrator/plans/ast/Job.scala | 36 +- .../orchestrator/plans/physical/ExecTask.scala | 4 +- .../plans/physical/RetryExecTask.scala | 2 +- .../orchestrator/reheater/AbstractReheater.scala | 29 +- .../linkis/orchestrator/reheater/Reheater.scala | 2 +- .../reheater/ReheaterNotifyTaskConsumer.scala | 22 +- .../plugin/linkis-orchestrator-ecm-plugin/pom.xml | 2 +- linkis-orchestrator/pom.xml | 2 +- .../linkis-bml/linkis-bml-client/pom.xml | 2 +- .../linkis-bml/linkis-bml-common/pom.xml | 2 +- .../linkis-bml/linkis-bml-engine-hook/pom.xml | 2 +- .../linkis-bml/linkis-bml-server/pom.xml | 2 +- .../linkis/bml/service/impl/TaskServiceImpl.java | 2 +- .../bml/service/impl/VersionServiceImpl.java | 3 +- linkis-public-enhancements/linkis-bml/pom.xml | 2 +- .../linkis-context-service/linkis-cs-cache/pom.xml | 2 +- .../cs/contextcache/cache/DefaultContextCache.java | 5 +- .../cache/csid/impl/ContextIDValueImpl.java | 5 +- .../linkis-cs-client/pom.xml | 2 +- .../cs/client/service/CSMetaDataService.java | 1 + .../cs/client/service/CSNodeServiceImpl.java | 1 + .../cs/client/service/CSResourceService.java | 2 + .../cs/client/service/CSResultDataServiceImpl.java | 6 + .../linkis/cs/client/service/CSTableService.java | 27 + .../cs/client/service/CSVariableService.java | 12 + .../cs/client/service/CSWorkServiceImpl.java | 1 + .../client/service/LinkisJobDataServiceImpl.java | 16 +- .../linkis-cs-common/pom.xml | 2 +- .../linkis-cs-highavailable/pom.xml | 2 +- .../cs/highavailable/DefaultContextHAManager.java | 6 +- .../highavailable/proxy/MethodInterceptorImpl.java | 7 +- .../linkis-cs-listener/pom.xml | 2 +- .../linkis-cs-persistence/pom.xml | 2 +- .../linkis-cs-search/pom.xml | 2 +- .../execution/ruler/NearestContextSearchRuler.java | 5 +- .../linkis-cs-server/pom.xml | 2 +- .../cs/server/service/impl/ContextServiceImpl.java | 4 + .../linkis-context-service/pom.xml | 2 +- .../linkis-datasource-client/pom.xml | 2 +- .../linkis-datasource-manager/common/pom.xml | 2 +- .../linkis-datasource-manager/server/pom.xml | 2 +- .../linkis-metadata-manager/common/pom.xml | 2 +- .../linkis-metadata-manager/server/pom.xml | 2 +- .../service/elasticsearch/pom.xml | 2 +- .../linkis-metadata-manager/service/hive/pom.xml | 7 +- .../metadatamanager/service/HiveMetaService.java | 3 + .../linkis-metadata-manager/service/kafka/pom.xml | 2 +- .../metadatamanager/service/KafkaMetaService.java | 3 + .../linkis-metadata-manager/service/mysql/pom.xml | 4 +- .../linkis-datasource/linkis-metadata/pom.xml | 2 +- .../linkis-datasource/pom.xml | 2 +- .../linkis-configuration/pom.xml | 2 +- .../linkis/configuration/dao/ConfigMapper.java | 6 +- .../linkis/configuration/dao/impl/ConfigMapper.xml | 36 +- .../exception/ConfigurationException.java | 14 +- .../restful/api/ConfigurationRestfulApi.java | 90 ++++ .../configuration/service/ConfigKeyService.java | 21 +- .../service/impl/ConfigKeyServiceImpl.java | 177 +++++++ .../service/ConfigurationService.scala | 127 +++-- .../configuration/util/LabelParameterParser.scala | 17 +- .../java/org/apache/linkis/configuration/Scan.java | 13 +- .../linkis/configuration/WebApplicationServer.java | 21 +- .../linkis/configuration/dao/BaseDaoTest.java | 18 +- .../linkis/configuration/dao/ConfigMapperTest.java | 193 ++++++++ .../linkis/configuration/dao/LabelMapperTest.java | 88 ++++ .../exception/ConfigurationExceptionTest.java | 7 +- .../restful/api/ConfigurationRestfulApiTest.java | 182 +++++++ .../linkis/configuration/restful/api/MvcUtils.java | 119 +++++ .../util/ConfigurationConfigurationTest.java | 25 +- .../configuration/util/LabelEntityParserTest.java | 57 +++ .../src/test/resources/application.properties | 56 +++ .../src/test/resources/create.sql | 89 ++++ .../src/test/resources/data.sql | 51 ++ .../src/test/resources/linkis.properties | 15 +- .../linkis-error-code-client/pom.xml | 2 +- .../linkis-error-code-common/pom.xml | 2 +- .../linkis-error-code-server/pom.xml | 2 +- .../linkis-publicservice/linkis-error-code/pom.xml | 2 +- .../linkis-instance-label-client/pom.xml | 2 +- .../linkis-instance-label-server/pom.xml | 2 +- .../instance/label/restful/InstanceRestful.java | 22 +- .../linkis-publicservice/linkis-jobhistory/pom.xml | 2 +- .../linkis/jobhistory/cache/QueryCacheManager.java | 2 + .../cache/impl/DefaultQueryCacheManager.java | 77 +++ .../cache/impl/ScheduledRefreshUndoneJob.java | 45 ++ .../linkis/jobhistory/dao/JobHistoryMapper.java | 15 +- .../jobhistory/dao/impl/JobHistoryMapper.xml | 15 + .../jobhistory/restful/api/QueryRestfulApi.java | 77 ++- .../jobhistory/conf/JobhistoryConfiguration.scala | 4 + .../jobhistory/service/JobHistoryQueryService.java | 2 +- .../service/impl/JobHistoryQueryServiceImpl.scala | 10 +- .../linkis-ps-common-lock/pom.xml | 2 +- .../linkis-storage-script-dev-client/pom.xml | 2 +- .../linkis-storage-script-dev-server/pom.xml | 2 +- .../filesystem/restful/api/FsRestfulApi.java | 6 +- .../linkis-udf/linkis-udf-client/pom.xml | 2 +- .../scala/org/apache/linkis/udf/UDFClient.scala | 12 +- .../linkis-udf/linkis-udf-common/pom.xml | 2 +- .../org/apache/linkis/udf/entity/UDFVersion.java | 17 + .../linkis-udf/linkis-udf-service/pom.xml | 2 +- .../java/org/apache/linkis/udf/dao/UDFTreeDao.java | 2 + .../org/apache/linkis/udf/dao/impl/UDFTreeDao.xml | 9 +- .../udf/service/impl/UDFTreeServiceImpl.java | 27 +- .../java/org/apache/linkis/udf/TestUDFTree.java | 43 -- .../linkis-publicservice/linkis-variable/pom.xml | 2 +- .../test/java/org/apache/linkis/variable/Scan.java | 13 +- .../linkis/variable/WebApplicationServer.java | 21 +- .../apache/linkis/variable/dao/BaseDaoTest.java | 18 +- .../apache/linkis/variable/dao/VarMapperTest.java | 107 ++++ .../restful/api/VariableRestfulApiTest.java | 39 +- .../variable/service/VariableServiceTest.java | 69 +++ .../src/test/resources/application.properties | 56 +++ .../linkis-variable/src/test/resources/create.sql | 45 ++ .../linkis-variable/src/test/resources/data.sql | 32 ++ .../linkis-publicservice/pom.xml | 2 +- linkis-public-enhancements/pom.xml | 2 +- .../linkis-service-discovery/linkis-eureka/pom.xml | 2 +- .../linkis-gateway-authentication/pom.xml | 2 +- .../gateway/authentication/dao/TokenDao.java | 4 +- .../authentication/dao/impl/TokenMapper.xml | 8 +- .../service/CachedTokenService.scala | 9 +- .../apache/linkis/gateway/authentication/Scan.java | 13 +- .../authentication/WebApplicationServer.java | 21 +- .../gateway/authentication/dao/BaseDaoTest.java} | 25 +- .../gateway/authentication/dao/TokenDaoTest.java} | 28 +- .../service/CachedTokenServiceTest.java | 82 +++ .../src/test/resources/application.properties | 62 +++ .../src/test/resources/create.sql | 19 +- .../src/test/resources/data.sql | 28 ++ .../src/test/resources/linkis.properties | 15 +- .../linkis-gateway-core/pom.xml | 2 +- .../gateway/config/GatewayConfiguration.scala | 3 + .../linkis/gateway/security/SecurityFilter.scala | 18 +- .../linkis-gateway-httpclient-support/pom.xml | 2 +- .../linkis-gateway-server-support/pom.xml | 2 +- .../linkis-spring-cloud-gateway/pom.xml | 2 +- .../SpringCloudGatewayConfiguration.scala | 17 +- .../http/LinkisGatewayHttpHeadersFilter.scala | 56 +++ .../linkis-gateway-datasource-ruler/pom.xml | 2 +- .../linkis-service-gateway/pom.xml | 2 +- linkis-spring-cloud-services/pom.xml | 2 +- pom.xml | 59 ++- tool/dependencies/known-dependencies.txt | 92 +++- web/.env | 2 +- web/config.sh | 12 +- web/install.sh | 79 ++- .../module/functionManagement/addFunctionModal.vue | 18 +- .../apps/URM/module/functionManagement/index.vue | 6 + .../URM/module/udfManagement/addFunctionModal.vue | 7 + web/src/apps/URM/module/udfManagement/index.vue | 6 + web/src/apps/linkis/module/ECM/engineConn.vue | 2 +- web/src/apps/linkis/module/ECM/index.vue | 2 +- .../linkis/module/resourceManagement/index.vue | 51 +- web/vue.config.js | 2 +- 365 files changed, 7223 insertions(+), 1393 deletions(-) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
