This is an automated email from the ASF dual-hosted git repository. aiceflower pushed a commit to branch release-0.9.4 in repository https://gitbox.apache.org/repos/asf/linkis.git
commit 7354d68091ed1cd63225504d26dd3e951bb0a75f Author: cooperyang <[email protected]> AuthorDate: Fri Jun 5 20:23:44 2020 +0800 make branch 0.9.4 to compile successfully (#439) make branch 0.9.4 to compile successfully (#439) --- assembly/pom.xml | 1 - assembly/public-module/pom.xml | 17 +++ assembly/src/main/assembly/assembly.xml | 34 +++++ bin/install.sh | 66 ++++----- bin/start-all.sh | 11 ++ bin/stop-all.sh | 10 ++ conf/config.sh | 9 ++ .../wedatasphere/linkis/cs/client/test/Test.java | 113 --------------- .../cs/client/test/bean/ClientTestContextKey.java | 68 ---------- .../client/test/bean/ClientTestContextValue.java | 37 ----- .../cs/client/test/bean/ResourceValueBean.java | 34 ----- .../test/listener/CommonContextKeyListener.java | 30 ---- .../cs/client/test/restful/ListenerTest.java | 8 -- .../linkis/cs/client/test/restful/RestfulTest.java | 92 ------------- .../linkis/cs/client/test/service/TestInfo.java | 83 ----------- .../linkis/cs/client/test/service/TestRemove.java | 114 ---------------- .../cs/client/test/service/TestSearchService.java | 112 --------------- .../test/test_multiuser/TestChangeContext.java | 94 ------------- .../linkis/protocol/query/RequestReadAllTask.java | 2 - .../linkis/protocol/query/ResponsePersist.java | 2 +- .../linkis/protocol/query/cache/CacheNotFound.java | 4 + .../protocol/query/cache/CacheTaskResult.java | 14 ++ .../protocol/query/cache/FailedToDeleteCache.java | 13 ++ .../protocol/query/cache/RequestDeleteCache.java | 28 ++++ .../protocol/query/cache/RequestReadCache.java | 33 +++++ .../protocol/query/cache/ResponseDeleteCache.java | 6 + .../protocol/query/cache/ResponseReadCache.java | 6 + .../protocol/query/cache/SuccessDeletedCache.java | 4 + .../protocol/usercontrol/RequestRegister.java | 2 - .../protocol/usercontrol/ResponseRegister.java | 2 +- db/linkis_ddl.sql | 74 ++++++++++ db/linkis_dml.sql | 6 +- db/module/linkis-datasource.sql | 67 +++++++++ metadata/bin/start-metadata.sh | 2 +- pom.xml | 151 +++++++++++---------- publicService/bin/start-publicservice.sh | 2 +- .../notify/NotifyRMEventSerializerTest.scala | 39 ------ .../resourcemanager/notify/ResourceTest.scala | 23 ---- .../resourcemanager/notify/ZookeeperLockTest.scala | 51 ------- .../resourcemanager/notify/ZookeeperPubTest.scala | 41 ------ .../notify/ZookeeperQueueTest.scala | 63 --------- .../resourcemanager/notify/ZookeeperSubTest.scala | 39 ------ .../src/test/scala/com/webank/test/TestMain.scala | 64 --------- .../test/scala/com/webank/test/TestResource.scala | 42 ------ storage/storage/pom.xml | 10 ++ .../linkis/storage/factory/BuildFactory.java | 1 - .../storage/factory/impl/BuildLocalFileSystem.java | 1 - .../wedatasphere/linkis/storage/fs/FileSystem.java | 1 - .../linkis/storage/fs/impl/LocalFileSystem.java | 2 - .../linkis/storage/pipeline/PipelineReader.java | 1 - .../linkis/storage/pipeline/PipelineWriter.java | 1 - .../wedatasphere/linkis/storage/FSFactory.scala | 1 - .../linkis/storage/domain/DataType.scala | 4 +- .../resultset/DefaultResultSetFactory.scala | 5 +- .../storage/resultset/StorageResultSet.scala | 2 +- .../storage/resultset/StorageResultSetReader.scala | 2 +- .../storage/resultset/StorageResultSetWriter.scala | 6 +- .../storage/resultset/html/HtmlResultSet.scala | 2 +- .../storage/resultset/io/IOResultSerializer.scala | 2 +- .../resultset/picture/PictureResultSet.scala | 4 +- .../resultset/txt/TextResultDeserializer.scala | 2 +- .../resultset/txt/TextResultSerializer.scala | 6 +- .../storage/resultset/txt/TextResultSet.scala | 2 +- .../linkis/storage/script/VariableParser.scala | 4 - .../script/compaction/QLScriptCompaction.scala | 1 + .../linkis/storage/source/AbstractFileSource.scala | 2 +- .../linkis/storage/source/FileSource.scala | 2 +- .../storage/source/ResultsetFileSource.scala | 2 +- .../linkis/storage/source/TextFileSource.scala | 2 +- .../linkis/storage/utils/StorageHelper.scala | 3 +- .../pipeline/conversions/FsConvertions.scala | 5 - .../spark/entrance/bin/start-spark-entrance.sh | 2 +- 72 files changed, 492 insertions(+), 1299 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index 11c9ae554c..57c23a49a1 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -26,7 +26,6 @@ <modelVersion>4.0.0</modelVersion> <artifactId>wedatasphere-linkis</artifactId> - <packaging>pom</packaging> <build> <plugins> <plugin> diff --git a/assembly/public-module/pom.xml b/assembly/public-module/pom.xml index 5cced71f20..a6b179ee95 100644 --- a/assembly/public-module/pom.xml +++ b/assembly/public-module/pom.xml @@ -1,4 +1,17 @@ <?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Copyright 2019 WeBank + ~ Licensed under the Apache License, Version 2.0 (the "License"); + ~ you may not use this file except in compliance with the License. + ~ You may obtain a copy of the License at + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + <project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> @@ -53,6 +66,10 @@ <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-util</artifactId> </exclusion> + <exclusion> + <artifactId>jackson-databind</artifactId> + <groupId>com.fasterxml.jackson.core</groupId> + </exclusion> </exclusions> </dependency> <dependency> diff --git a/assembly/src/main/assembly/assembly.xml b/assembly/src/main/assembly/assembly.xml index 9c8a032b9d..ec4b928890 100644 --- a/assembly/src/main/assembly/assembly.xml +++ b/assembly/src/main/assembly/assembly.xml @@ -268,8 +268,42 @@ <include>*.zip</include> </includes> </fileSet> + <!-- context service--> + <fileSet> + <directory> + ${project.parent.basedir}/contextservice/cs-server/target/ + </directory> + <outputDirectory>share/linkis/linkis-cs-server/</outputDirectory> + <includes> + <include>*.zip</include> + </includes> + </fileSet> + <!--datasource manage server--> + <fileSet> + <directory> + ${project.parent.basedir}/datasource/datasourcemanager/server/target + </directory> + <outputDirectory>share/linkis/datasource/linkis-dsm-server/</outputDirectory> + <includes> + <include>*.zip</include> + </includes> + </fileSet> + + + <!--metadata manage server--> + <fileSet> + <directory> + ${project.parent.basedir}/datasource/metadatamanager/server/target + </directory> + <outputDirectory>share/linkis/datasource/linkis-mdm-server/</outputDirectory> + <includes> + <include>*.zip</include> + </includes> + </fileSet> </fileSets> + + <dependencySets> <dependencySet> <outputDirectory>lib</outputDirectory> diff --git a/bin/install.sh b/bin/install.sh index 9fcf298667..b646689abf 100644 --- a/bin/install.sh +++ b/bin/install.sh @@ -122,35 +122,6 @@ source ${DISTRIBUTION} isSuccess "load config" -local_host="`hostname --fqdn`" - -ipaddr=$(ip addr | awk '/^[0-9]+: / {}; /inet.*global/ {print gensub(/(.*)\/(.*)/, "\\1", "g", $2)}'|awk 'NR==1') - -function isLocal(){ - if [ "$1" == "127.0.0.1" ];then - return 0 - elif [ $1 == "localhost" ]; then - return 0 - elif [ $1 == $local_host ]; then - return 0 - elif [ $1 == $ipaddr ]; then - return 0 - fi - return 1 -} - -function executeCMD(){ - isLocal $1 - flag=$? - if [ $flag == "0" ];then - echo "Is local execution:$2" - eval $2 - else - echo "Is remote execution:$2" - ssh -p $SSH_PORT $1 $2 - fi - - ##install mode choice if [ "$INSTALL_MODE" == "" ];then @@ -164,7 +135,7 @@ if [ "$INSTALL_MODE" == "" ];then fi if [[ '1' = "$INSTALL_MODE" ]];then - echo "You chose Lite installation mode" + echo "You chose Lite installation mode" checkPythonAndJava elif [[ '2' = "$INSTALL_MODE" ]];then echo "You chose Simple installation mode" @@ -247,7 +218,7 @@ then localRootDir=${RESULT_SET_ROOT_PATH#hdfs://} hdfs dfs -mkdir -p $localRootDir/$deployUser else - echo "does not support $RESULT_SET_ROOT_PATH filesystem types" + echo "does not support $RESULT_SET_ROOT_PATH filesystem types" fi fi isSuccess "create $RESULT_SET_ROOT_PATH directory" @@ -673,4 +644,37 @@ echo "<----------------$SERVER_NAME:end------------------->" ##SHELLEntrance install end +##Datasource Manager Server install +PACKAGE_DIR=linkis/datasource/linkis-dsm-server +SERVER_NAME=linkis-dsm-server +SERVER_PORT=$DSM_PORT +###install dir +installPackage +###update linkis.properties +echo "$SERVER_NAME-step4:update linkis conf" +SERVER_CONF_PATH=$SERVER_HOME/$SERVER_NAME/conf/linkis.properties +executeCMD $SERVER_IP "sed -i ${txt} \"s#wds.linkis.server.mybatis.datasource.url.*#wds.linkis.server.mybatis.datasource.url=jdbc:mysql://${MYSQL_HOST}:${MYSQL_PORT}/${MYSQL_DB}?characterEncoding=UTF-8#g\" $SERVER_CONF_PATH" +executeCMD $SERVER_IP "sed -i ${txt} \"s#wds.linkis.server.mybatis.datasource.username.*#wds.linkis.server.mybatis.datasource.username=$MYSQL_USER#g\" $SERVER_CONF_PATH" +executeCMD $SERVER_IP "sed -i ${txt} \"s#wds.linkis.server.mybatis.datasource.password.*#wds.linkis.server.mybatis.datasource.password=$MYSQL_PASSWORD#g\" $SERVER_CONF_PATH" +executeCMD $SERVER_IP "sed -i ${txt} \"s#wds.linkis.server.dsm.admin.users.*#wds.linkis.server.dsm.admin.users=$deployUser#g\" $SERVER_CONF_PATH" +replaceConf "wds.linkis.gateway.url" "http://$GATEWAY_INSTALL_IP:$GATEWAY_PORT"; "$SERVER_CONF_PATH" +isSuccess "subsitution linkis.properties of $SERVER_NAME" +echo "<----------------$SERVER_NAME:end------------------->" +##Datasource Manager Server install end + + + +##Metadata Manager Server install +PACKAGE_DIR=linkis/datasource/linkis-mdm-server +SERVER_NAME=linkis-mdm-server +SERVER_PORT=$MDM_PORT +###install dir +installPackage +###update linkis.properties +echo "$SERVER_NAME-step4:update linkis conf" +SERVER_CONF_PATH=$SERVER_HOME/$SERVER_NAME/conf/linkis.properties +replaceConf "wds.linkis.gateway.url" "http://$GATEWAY_INSTALL_IP:$GATEWAY_PORT"; "$SERVER_CONF_PATH" +isSuccess "subsitution linkis.properties of $SERVER_NAME" +echo "<----------------$SERVER_NAME:end------------------->" +##Metadata Manager Server install end diff --git a/bin/start-all.sh b/bin/start-all.sh index 822172c951..05f5046984 100644 --- a/bin/start-all.sh +++ b/bin/start-all.sh @@ -83,6 +83,7 @@ function startApp(){ echo "<-------------------------------->" echo "Begin to start $SERVER_NAME" SERVER_PATH=${APP_PREFIX}${SERVER_NAME} + SERVER_BIN=${LINKIS_INSTALL_HOME}/${SERVER_PATH}/bin SERVER_LOCAL_START_CMD="dos2unix ${SERVER_BIN}/* > /dev/null 2>&1; dos2unix ${SERVER_BIN}/../conf/* > /dev/null 2>&1; sh ${SERVER_BIN}/start-${SERVER_NAME}.sh" SERVER_REMOTE_START_CMD="source /etc/profile;source ~/.bash_profile;cd ${SERVER_BIN}; dos2unix ./* > /dev/null 2>&1; dos2unix ../conf/* > /dev/null 2>&1; sh start-${SERVER_NAME}.sh > /dev/null 2>&1" @@ -144,6 +145,16 @@ SERVER_NAME="cs-server" SERVER_IP=$CS_INSTALL_IP startApp +#datasource management +SERVER_NAME="dsm-server" +SERVER_IP=$DSM_INSTALL_IP +startApp + +#metadata management +SERVER_NAME="mdm-server" +SERVER_IP=$MDM_INSTALL_IP +startApp + #resourcemanager SERVER_NAME="resourcemanager" SERVER_IP=$RESOURCEMANAGER_INSTALL_IP diff --git a/bin/stop-all.sh b/bin/stop-all.sh index b9dc14dd79..098a5772b7 100644 --- a/bin/stop-all.sh +++ b/bin/stop-all.sh @@ -140,6 +140,16 @@ SERVER_NAME="cs-server" SERVER_IP=$CS_INSTALL_IP stopApp +#datasource management +SERVER_NAME="dsm-server" +SERVER_IP=$DSM_INSTALL_IP +stopApp + +#metadata management +SERVER_NAME="mdm-server" +SERVER_IP=$MDM_INSTALL_IP +stopApp + APP_PREFIX="linkis-ujes-" #python-entrance diff --git a/conf/config.sh b/conf/config.sh index 619752ab5c..ef8a848bc7 100644 --- a/conf/config.sh +++ b/conf/config.sh @@ -114,6 +114,15 @@ BML_PORT=9113 #CS_INSTALL_IP=127.0.0.1 CS_PORT=9116 + +### datasource management server +#DSM_INSTALL_IP=127.0.0.1 +DSM_PORT=9117 + +### metadata management server +#MDM_INSTALL_IP=127.0.0.1 +MDM_PORT=9118 + ######################################################################################## ## LDAP is for enterprise authorization, if you just want to have a try, ignore it. diff --git a/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/Test.java b/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/Test.java deleted file mode 100644 index 53638e1cfa..0000000000 --- a/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/Test.java +++ /dev/null @@ -1,113 +0,0 @@ -package com.webank.wedatasphere.linkis.cs.client.test; - -import com.google.gson.Gson; -import com.webank.wedatasphere.linkis.common.listener.Event; -import com.webank.wedatasphere.linkis.cs.client.Context; -import com.webank.wedatasphere.linkis.cs.client.ContextClient; -import com.webank.wedatasphere.linkis.cs.client.builder.ContextClientFactory; -import com.webank.wedatasphere.linkis.cs.client.listener.ContextClientListener; -import com.webank.wedatasphere.linkis.cs.client.listener.ContextClientListenerBus; -import com.webank.wedatasphere.linkis.cs.client.listener.ContextClientListenerManager; -import com.webank.wedatasphere.linkis.cs.client.listener.ContextKeyListener; -import com.webank.wedatasphere.linkis.cs.client.test.listener.CommonContextKeyListener; -import com.webank.wedatasphere.linkis.cs.client.utils.SerializeHelper; -import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope; -import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType; -import com.webank.wedatasphere.linkis.cs.common.entity.object.CSFlowInfos; -import com.webank.wedatasphere.linkis.cs.common.entity.resource.LinkisBMLResource; -import com.webank.wedatasphere.linkis.cs.common.entity.source.*; -import com.webank.wedatasphere.linkis.cs.common.serialize.helper.ContextSerializationHelper; - -import java.util.ArrayList; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * created by cooperyang on 2020/2/24 - * Description: - */ -public class Test { - public static void main(String[] args)throws Exception{ - /*ContextClient contextClient = ContextClientFactory.getOrCreateContextClient(); - ContextID contextID = new LinkisHAWorkFlowContextID(); - Context context = contextClient.createContext(contextID); - System.out.println(context.getContextID().getContextId()); - ContextKey contextKey = new CommonContextKey(); - contextKey.setKey("cooper.txt"); - contextKey.setKeywords("xddd"); - contextKey.setContextScope(ContextScope.PUBLIC); - contextKey.setContextType(ContextType.RESOURCE); - ContextValue contextValue = new CommonContextValue(); - LinkisBMLResource resource = new LinkisBMLResource(); - resource.setResourceId("456789"); - resource.setVersion("v00001"); - contextValue.setValue(resource); - ContextKeyValue contextKeyValue = new CommonContextKeyValue(); - contextKeyValue.setContextValue(contextValue); - contextKeyValue.setContextKey(contextKey); - context.setContextKeyAndValue(contextKeyValue); - ContextValue myValue = context.getContextValue(contextKey); - LinkisBMLResource linkisBMLResource = (LinkisBMLResource)myValue.getValue(); - System.out.println(linkisBMLResource.getResourceId()); - ContextClientListenerBus<ContextClientListener, Event> bus = ContextClientListenerManager.getContextClientListenerBus(); - ContextKeyListener listener = new CommonContextKeyListener(); - listener.setContextKey(contextKey); - listener.setContext(context); - context.onBind(contextKey, listener); - bus.addListener(listener); - List<String> contains = new ArrayList<>(); - contains.add("cooper"); - contextClient.search(context.getContextID(), null, null, contains, null); - List<ContextKeyValue> list = context.searchContext(null, null, contains, null); - System.out.println("ret list is " + list); - Thread.sleep(5000); - contextClient.close();*/ - testCSFlowInfos(); - - } - - public static void testCSFlowInfos() throws Exception { - ContextClient contextClient = ContextClientFactory.getOrCreateContextClient(); - LinkisHAWorkFlowContextID contextID = new LinkisHAWorkFlowContextID(); - contextID.setFlow("test"); - contextID.setProject("test"); - contextID.setWorkSpace("test1"); - contextID.setVersion("v00001"); - Context context = contextClient.createContext(contextID); - System.out.println(context.getContextID().getContextId()); - - ContextSerializationHelper contextSerializationHelper = ContextSerializationHelper.getInstance(); - CommonContextValue contextValue = new CommonContextValue(); - Map<String, Object> infos = new HashMap<>(); - List<Map<String, String>> edges = new ArrayList<>(); - Map<String, String> edge = new HashMap<>(); - edge.put("source","sql " ); - edge.put("target","hql"); - edge.put("sourceLocation","bottom"); - edge.put("targetLocation","top"); - edges.add(edge); - infos.put("edges", edges); - infos.put("parent", "flow2"); - Map<String, String> idNodeName = new HashMap<>(); - idNodeName.put("90a6ee94-4bd6-47d9-a536-f92660c4c051", "sql"); - idNodeName.put("90a6ee94-4bd6-47d9-a536-f92660c4c052", "hql"); - infos.put("id_nodeName", idNodeName); - CSFlowInfos csFlowInfos = new CSFlowInfos(); - csFlowInfos.setInfos(infos); - contextValue.setValue(csFlowInfos); - - ContextKey contextKey = new CommonContextKey(); - contextKey.setKey("flow.infos"); - contextKey.setContextScope(ContextScope.PUBLIC); - contextKey.setContextType(ContextType.RESOURCE); - - ContextKeyValue contextKeyValue = new CommonContextKeyValue(); - contextKeyValue.setContextValue(contextValue); - contextKeyValue.setContextKey(contextKey); - context.setContextKeyAndValue(contextKeyValue); - ContextValue myValue = context.getContextValue(contextKey); - System.out.println(SerializeHelper.serializeContextID(context.getContextID())); - } -} diff --git a/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/bean/ClientTestContextKey.java b/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/bean/ClientTestContextKey.java deleted file mode 100644 index f9edabb3f0..0000000000 --- a/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/bean/ClientTestContextKey.java +++ /dev/null @@ -1,68 +0,0 @@ -package com.webank.wedatasphere.linkis.cs.client.test.bean; - -import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope; -import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType; -import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey; - -/** - * created by cooperyang on 2020/2/26 - * Description: - */ -public class ClientTestContextKey implements ContextKey { - - - private final String key = "cooperyang.txt"; - - - - - @Override - public String getKey() { - return this.key; - } - - @Override - public void setKey(String key) { - - } - - @Override - public int getType() { - return 0; - } - - @Override - public void setType(int type) { - - } - - @Override - public ContextType getContextType() { - return null; - } - - @Override - public void setContextType(ContextType contextType) { - - } - - @Override - public ContextScope getContextScope() { - return null; - } - - @Override - public void setContextScope(ContextScope contextScope) { - - } - - @Override - public String getKeywords() { - return null; - } - - @Override - public void setKeywords(String keywords) { - - } -} diff --git a/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/bean/ClientTestContextValue.java b/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/bean/ClientTestContextValue.java deleted file mode 100644 index d0f11e0aef..0000000000 --- a/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/bean/ClientTestContextValue.java +++ /dev/null @@ -1,37 +0,0 @@ -package com.webank.wedatasphere.linkis.cs.client.test.bean; - -import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextValue; -import com.webank.wedatasphere.linkis.cs.common.entity.source.ValueBean; - -/** - * created by cooperyang on 2020/2/26 - * Description: - */ -public class ClientTestContextValue implements ContextValue { - - - - private Object value; - - private String keywords; - - @Override - public String getKeywords() { - return this.keywords; - } - - @Override - public void setKeywords(String keywords) { - this.keywords = keywords; - } - - @Override - public Object getValue() { - return this.value; - } - - @Override - public void setValue(Object value) { - this.value = value; - } -} diff --git a/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/bean/ResourceValueBean.java b/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/bean/ResourceValueBean.java deleted file mode 100644 index 8632b0324b..0000000000 --- a/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/bean/ResourceValueBean.java +++ /dev/null @@ -1,34 +0,0 @@ -package com.webank.wedatasphere.linkis.cs.client.test.bean; - -import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType; -import com.webank.wedatasphere.linkis.cs.common.entity.resource.Resource; - -/** - * created by cooperyang on 2020/2/26 - * Description: - */ -public class ResourceValueBean implements Resource { - - private String resourceId; - private String version; - - private final ContextType contextType = ContextType.RESOURCE; - - public String getResourceId() { - return resourceId; - } - - public void setResourceId(String resourceId) { - this.resourceId = resourceId; - } - - public String getVersion() { - return version; - } - - public void setVersion(String version) { - this.version = version; - } - - -} diff --git a/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/listener/CommonContextKeyListener.java b/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/listener/CommonContextKeyListener.java deleted file mode 100644 index 40cb99b11c..0000000000 --- a/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/listener/CommonContextKeyListener.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.webank.wedatasphere.linkis.cs.client.test.listener; - -import com.webank.wedatasphere.linkis.common.listener.Event; -import com.webank.wedatasphere.linkis.cs.client.listener.ContextKeyListener; - -/** - * created by cooperyang on 2020/3/1 - * Description: - */ -public class CommonContextKeyListener extends ContextKeyListener { - - - - - @Override - public void onContextCreated(Event event) { - System.out.println("it is created"); - } - - @Override - public void onContextUpdated(Event event) { - super.onContextUpdated(event); - System.out.println("it is updated"); - } - - @Override - public void onEventError(Event event, Throwable t) { - System.out.println("it is not ok"); - } -} diff --git a/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/restful/ListenerTest.java b/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/restful/ListenerTest.java deleted file mode 100644 index e28baeff5f..0000000000 --- a/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/restful/ListenerTest.java +++ /dev/null @@ -1,8 +0,0 @@ -package com.webank.wedatasphere.linkis.cs.client.test.restful; - -/** - * created by cooperyang on 2020/3/1 - * Description: - */ -public class ListenerTest { -} diff --git a/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/restful/RestfulTest.java b/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/restful/RestfulTest.java deleted file mode 100644 index 6e52e0d57d..0000000000 --- a/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/restful/RestfulTest.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright 2019 WeBank - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.webank.wedatasphere.linkis.cs.client.test.restful; - -import com.webank.wedatasphere.linkis.common.exception.ErrorException; -import com.webank.wedatasphere.linkis.cs.client.ContextClient; -import com.webank.wedatasphere.linkis.cs.client.builder.ContextClientFactory; -import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope; -import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType; -import com.webank.wedatasphere.linkis.cs.common.entity.resource.LinkisBMLResource; -import com.webank.wedatasphere.linkis.cs.common.entity.resource.Resource; -import com.webank.wedatasphere.linkis.cs.common.entity.source.*; - -/** - * Created by v_wbjftang on 2020/2/28. - */ -public class RestfulTest { - /** - * setValueByKey - */ - ContextClient contextClient = null; - - ContextID contextID = null; - - ContextKey contextKey = null; - - ContextValue contextValue = null; - - ContextKeyValue keyValue = null; - - - Resource resource = new LinkisBMLResource(); - - /*@Before*/ - public void init() { - contextClient = ContextClientFactory.getOrCreateContextClient(); - contextID = new LinkisHAWorkFlowContextID(); - contextID.setContextId("84716"); - - contextKey = new CommonContextKey(); - contextKey.setContextScope(ContextScope.FRIENDLY); - contextKey.setContextType(ContextType.ENV); - contextKey.setKey("project1.flow1.node1.key2"); - - contextValue = new CommonContextValue(); - LinkisBMLResource resource = new LinkisBMLResource(); - resource.setResourceId("dfasdfsr2456wertg"); - resource.setVersion("v000002"); - contextValue.setValue(resource); - - keyValue = new CommonContextKeyValue(); - keyValue.setContextKey(contextKey); - keyValue.setContextValue(contextValue); - - resource.setResourceId("edtr44356-34563456"); - resource.setVersion("v000004"); - - - } - - /** - * setValueByKey - * - * @throws ErrorException - */ - /* @org.junit.Test*/ - public void test01() throws ErrorException { - contextClient.update(contextID, contextKey, contextValue); - } - - /** - * reset - * - * @throws ErrorException - */ - /* @org.junit.Test*/ - public void test02() throws ErrorException { - contextClient.reset(contextID, contextKey); - } - -} diff --git a/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/service/TestInfo.java b/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/service/TestInfo.java deleted file mode 100644 index 30afc0e251..0000000000 --- a/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/service/TestInfo.java +++ /dev/null @@ -1,83 +0,0 @@ -package com.webank.wedatasphere.linkis.cs.client.test.service; - -import com.google.gson.GsonBuilder; -import com.webank.wedatasphere.linkis.cs.client.Context; -import com.webank.wedatasphere.linkis.cs.client.ContextClient; -import com.webank.wedatasphere.linkis.cs.client.builder.ContextClientFactory; -//import com.webank.wedatasphere.linkis.cs.client.service.DefaultSearchService; -import com.webank.wedatasphere.linkis.cs.client.service.DefaultSearchService; -import com.webank.wedatasphere.linkis.cs.client.service.SearchService; -import com.webank.wedatasphere.linkis.cs.client.utils.SerializeHelper; -import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope; -import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType; -import com.webank.wedatasphere.linkis.cs.common.entity.object.CSFlowInfos; -import com.webank.wedatasphere.linkis.cs.common.entity.resource.BMLResource; -import com.webank.wedatasphere.linkis.cs.common.entity.resource.LinkisBMLResource; -import com.webank.wedatasphere.linkis.cs.common.entity.source.*; -import com.webank.wedatasphere.linkis.cs.common.serialize.helper.ContextSerializationHelper; -import com.webank.wedatasphere.linkis.cs.common.utils.CSCommonUtils; -//import com.webank.wedatasphere.linkis.cs.common.utils.CSCommonUtils; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * @Author alexyang - * @Date 2020/3/16 - */ -public class TestInfo { - - - public static void main(String [] args) throws Exception { - - ContextClient contextClient = ContextClientFactory.getOrCreateContextClient(); - LinkisHAWorkFlowContextID contextID = new LinkisHAWorkFlowContextID(); - contextID.setFlow("test"); - contextID.setProject("test"); - contextID.setWorkSpace("test1"); - contextID.setVersion("v00001"); - Context context = contextClient.createContext(contextID); - System.out.println(context.getContextID().getContextId()); - - ContextSerializationHelper contextSerializationHelper = ContextSerializationHelper.getInstance(); - CommonContextValue contextValue = new CommonContextValue(); - Map<String, Object> infos = new HashMap<>(); - List<Map<String, String>> edges = new ArrayList<>(); - Map<String, String> edge = new HashMap<>(); - edge.put("source","sql " ); - edge.put("target","hql"); - edge.put("sourceLocation","bottom"); - edge.put("targetLocation","top"); - edges.add(edge); - infos.put("edges", edges); - infos.put("parent", "flow2"); - Map<String, String> idNodeName = new HashMap<>(); - idNodeName.put("90a6ee94-4bd6-47d9-a536-f92660c4c051", "sql"); - idNodeName.put("90a6ee94-4bd6-47d9-a536-f92660c4c052", "hql"); - infos.put("id_nodeName", idNodeName); - CSFlowInfos csFlowInfos = new CSFlowInfos(); - csFlowInfos.setInfos(infos); - contextValue.setValue(csFlowInfos); - - ContextKey contextKey = new CommonContextKey(); - contextKey.setKey("flow.infos"); - contextKey.setContextScope(ContextScope.PUBLIC); - contextKey.setContextType(ContextType.RESOURCE); - - ContextKeyValue contextKeyValue = new CommonContextKeyValue(); - contextKeyValue.setContextValue(contextValue); - contextKeyValue.setContextKey(contextKey); - context.setContextKeyAndValue(contextKeyValue); - ContextValue myValue = context.getContextValue(contextKey); - System.out.println(SerializeHelper.serializeContextID(context.getContextID())); - System.out.println(CSCommonUtils.gson.toJson(myValue)); - - SearchService searchService = DefaultSearchService.getInstance(); - BMLResource bmlResource = searchService.searchContext(contextID, "ql", "sql", LinkisBMLResource.class); - System.out.println(new GsonBuilder().setPrettyPrinting().create().toJson(bmlResource)); - - contextClient.close(); - } -} diff --git a/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/service/TestRemove.java b/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/service/TestRemove.java deleted file mode 100644 index 4f296af6c2..0000000000 --- a/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/service/TestRemove.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright 2019 WeBank - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.webank.wedatasphere.linkis.cs.client.test.service; - -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.webank.wedatasphere.linkis.common.exception.ErrorException; -import com.webank.wedatasphere.linkis.cs.client.Context; -import com.webank.wedatasphere.linkis.cs.client.ContextClient; -import com.webank.wedatasphere.linkis.cs.client.builder.ContextClientFactory; -import com.webank.wedatasphere.linkis.cs.client.service.CSWorkService; -import com.webank.wedatasphere.linkis.cs.client.service.CSWorkServiceImpl; -import com.webank.wedatasphere.linkis.cs.client.utils.SerializeHelper; -import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope; -import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType; -import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.WorkType; -import com.webank.wedatasphere.linkis.cs.common.entity.resource.LinkisBMLResource; -import com.webank.wedatasphere.linkis.cs.common.entity.source.*; - -import static org.junit.Assert.assertEquals; - -/** - * @Author alexyang - * @Date 2020/3/21 - */ -public class TestRemove { - - private ContextClient contextClient; - private ContextID contextId; - private ContextKey contextKey; - private ContextValue contextValue; - private CSWorkService csWorkService; - private Gson gson; - - public static void main(String [] args) throws Exception { - TestRemove test = new TestRemove(); - test.init(); - test.testGetBeforeSet(); - test.testReset(); - test.testGetAfterSet(); - test.testClose(); - } - - - public void init() throws Exception { - contextClient = ContextClientFactory.getOrCreateContextClient(); - csWorkService = CSWorkServiceImpl.getInstance(); - gson = new GsonBuilder().setPrettyPrinting().create(); - LinkisHAWorkFlowContextID haContextID = new LinkisHAWorkFlowContextID(); - haContextID.setWorkSpace("wp1"); - haContextID.setProject("p1"); - haContextID.setFlow("f1"); - haContextID.setVersion("v1"); - if (null == contextId) { - Context context = contextClient.createContext(haContextID); - contextId = context.getContextID(); - } - - contextKey = new CommonContextKey(); - contextKey.setContextScope(ContextScope.FRIENDLY); - contextKey.setContextType(ContextType.ENV); - contextKey.setKey("project.flow1.node1.key2"); - - contextValue = new CommonContextValue(); - LinkisBMLResource resource = new LinkisBMLResource(); - resource.setResourceId("dfasdfsr2456wertg"); - resource.setVersion("v000002"); - contextValue.setValue(resource); - - ContextKeyValue keyValue = new CommonContextKeyValue(); - keyValue.setContextKey(contextKey); - keyValue.setContextValue(contextValue); - - contextClient.setContextKeyValue(haContextID, keyValue); - - } - - - public void testGetBeforeSet() throws ErrorException { - ContextValue value = contextClient.getContextValue(contextId, contextKey); - LinkisBMLResource resourceOri = (LinkisBMLResource)contextValue.getValue(); - LinkisBMLResource resourceRs = (LinkisBMLResource)value.getValue(); - assertEquals(resourceOri.getResourceId(), resourceRs.getResourceId()); - assertEquals(resourceOri.getVersion(), resourceRs.getVersion()); - System.out.println(gson.toJson(resourceOri)); - System.out.println(gson.toJson(resourceRs)); - } - - public void testReset() throws ErrorException { - csWorkService.initContextServiceInfo(SerializeHelper.serializeContextID(contextId), WorkType.PROJECT); - } - - public void testGetAfterSet() throws ErrorException { - ContextValue value = contextClient.getContextValue(contextId, contextKey); - assertEquals(null, value); - System.out.println(gson.toJson(contextValue)); - System.out.println(gson.toJson(value)); - } - - public void testClose() throws Exception { - contextClient.close(); - } -} diff --git a/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/service/TestSearchService.java b/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/service/TestSearchService.java deleted file mode 100644 index 46080a63f1..0000000000 --- a/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/service/TestSearchService.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright 2019 WeBank - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.webank.wedatasphere.linkis.cs.client.test.service; - -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.webank.wedatasphere.linkis.cs.client.Context; -import com.webank.wedatasphere.linkis.cs.client.ContextClient; -import com.webank.wedatasphere.linkis.cs.client.builder.ContextClientFactory; -import com.webank.wedatasphere.linkis.cs.client.service.DefaultSearchService; -import com.webank.wedatasphere.linkis.cs.client.service.SearchService; -import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope; -import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType; -import com.webank.wedatasphere.linkis.cs.common.entity.object.CSFlowInfos; -import com.webank.wedatasphere.linkis.cs.common.entity.resource.BMLResource; -import com.webank.wedatasphere.linkis.cs.common.entity.resource.LinkisBMLResource; -import com.webank.wedatasphere.linkis.cs.common.entity.source.*; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * @Author alexyang - * @Date 2020/3/8 - */ -public class TestSearchService { - - public static void main(String [] args) throws Exception { - - Gson gson = new GsonBuilder().setPrettyPrinting().create(); - SearchService searchService = DefaultSearchService.getInstance(); - ContextClient contextClient = ContextClientFactory.getOrCreateContextClient(); - LinkisHAWorkFlowContextID contextID = new LinkisHAWorkFlowContextID(); - { - contextID.setUser("test"); - contextID.setProject("test_client"); - contextID.setFlow("test_client"); - Context context = contextClient.createContext(contextID); - contextID = (LinkisHAWorkFlowContextID) context.getContextID(); - System.out.println(new GsonBuilder().setPrettyPrinting().create().toJson(contextID)); - ContextKey contextKey = new CommonContextKey(); - contextKey.setKey("node.resource.sql.cooper.txt"); - contextKey.setContextType(ContextType.RESOURCE); - contextKey.setContextScope(ContextScope.PUBLIC); -// contextKey.setKeywords("xddd"); - ContextValue contextValue = new CommonContextValue(); - LinkisBMLResource linkisBMLResource = new LinkisBMLResource(); - linkisBMLResource.setVersion("v000001"); - linkisBMLResource.setDownloadedFileName("t.txt"); - linkisBMLResource.setResourceId("adsf"); - contextValue.setValue(linkisBMLResource); - ContextKeyValue contextKeyValue = new CommonContextKeyValue(contextKey, contextValue); - contextClient.setContextKeyValue(contextID, contextKeyValue); - BMLResource resource = searchService.getContextValue(contextID, contextKey, LinkisBMLResource.class); - System.out.println(gson.toJson(resource)); - ContextValue contextValue1 = contextClient.getContextValue(contextID, contextKey); - System.out.println(contextValue1.getValue()); - - } - - { - CommonContextValue contextValue = new CommonContextValue(); - Map<String, Object> infos = new HashMap<>(); - List<Map<String, String>> edges = new ArrayList<>(); - Map<String, String> edge = new HashMap<>(); - edge.put("source","90a6ee94-4bd6-47d9-a536-f92660c4c051" ); - edge.put("target","90a6ee94-4bd6-47d9-a536-f92660c4c052"); - edge.put("sourceLocation","bottom"); - edge.put("targetLocation","top"); - edges.add(edge); - infos.put("edges", edges); - infos.put("parent", "flow2"); - Map<String, String> idNodeName = new HashMap<>(); - idNodeName.put("90a6ee94-4bd6-47d9-a536-f92660c4c051", "sql"); - idNodeName.put("90a6ee94-4bd6-47d9-a536-f92660c4c052", "hql"); - infos.put("id_nodeName", idNodeName); - CSFlowInfos csFlowInfos = new CSFlowInfos(); - csFlowInfos.setInfos(infos); - contextValue.setValue(csFlowInfos); - - ContextKey contextKey = new CommonContextKey(); - contextKey.setKey("flow.infos"); - contextKey.setContextScope(ContextScope.PUBLIC); - contextKey.setContextType(ContextType.OBJECT); - - ContextKeyValue contextKeyValue = new CommonContextKeyValue(); - contextKeyValue.setContextValue(contextValue); - contextKeyValue.setContextKey(contextKey); - contextClient.setContextKeyValue(contextID, contextKeyValue); - Object flowInfoObject = searchService.getContextValue(contextID, contextKey, Object.class); - System.out.println(gson.toJson(flowInfoObject)); - } - - BMLResource bmlResource = searchService.searchContext(contextID, "cooper", "sql", LinkisBMLResource.class); - contextClient.close(); - System.out.println(gson.toJson(bmlResource)); - } - -} \ No newline at end of file diff --git a/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/test_multiuser/TestChangeContext.java b/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/test_multiuser/TestChangeContext.java deleted file mode 100644 index 08a653f434..0000000000 --- a/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/test_multiuser/TestChangeContext.java +++ /dev/null @@ -1,94 +0,0 @@ -package com.webank.wedatasphere.linkis.cs.client.test.test_multiuser; - -import com.google.gson.Gson; -import com.webank.wedatasphere.linkis.cs.client.Context; -import com.webank.wedatasphere.linkis.cs.client.ContextClient; -import com.webank.wedatasphere.linkis.cs.client.builder.ContextClientFactory; -import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope; -import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType; -import com.webank.wedatasphere.linkis.cs.common.entity.resource.LinkisBMLResource; -import com.webank.wedatasphere.linkis.cs.common.entity.source.*; -import com.webank.wedatasphere.linkis.cs.common.serialize.helper.ContextSerializationHelper; -import com.webank.wedatasphere.linkis.cs.common.serialize.helper.SerializationHelper; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; - -/** - * @Author alexyang - * @Date 2020/2/28 - */ -public class TestChangeContext { - - public static void main(String [] args) { - - ContextClient contextClient = ContextClientFactory.getOrCreateContextClient(); - try { - // 1, read context - File file = new File(TestCreateContext.CONTEXTID_PATH); - if (!file.exists()) { - System.out.println("Error, contextID serialize file : " + TestCreateContext.CONTEXTID_PATH + " invalid."); - return; - } - FileReader fr = new FileReader(file); - BufferedReader br = new BufferedReader(fr); - String contextIDStr = null; - StringBuilder builder = new StringBuilder(""); - java.lang.String tmp = br.readLine(); - while (null != tmp) { - builder.append(tmp); - tmp = br.readLine(); - } - br.close(); - contextIDStr = builder.toString(); - System.out.println("Read contextID : " + contextIDStr); - - SerializationHelper serializationHelper = ContextSerializationHelper.getInstance(); - if (!serializationHelper.accepts(contextIDStr)) { - System.out.println("Invalid contextStr : " + contextIDStr + ", cannot be deserialized"); - return; - } - Object contextObject = serializationHelper.deserialize(contextIDStr); - System.out.println("Deserialized obj : " + new Gson().toJson(contextObject)); - if (!LinkisHAWorkFlowContextID.class.isInstance(contextObject)) { - System.out.println("Invalid contextObject, not LinkisHAWorkFlowContextID instance."); - return; - } - LinkisHAWorkFlowContextID contextID = (LinkisHAWorkFlowContextID) contextObject; - - Context context = contextClient.getContext(contextID); - // 2, update context - ContextKey contextKey = new CommonContextKey(); - contextKey.setKey("testchange.txt"); - contextKey.setKeywords("xddd"); - contextKey.setContextScope(ContextScope.PUBLIC); - contextKey.setContextType(ContextType.RESOURCE); - ContextValue contextValue = new CommonContextValue(); - LinkisBMLResource resource = new LinkisBMLResource(); - resource.setResourceId("456789"); - resource.setVersion("v00001"); - contextValue.setValue(resource); - ContextKeyValue contextKeyValue = new CommonContextKeyValue(); - contextKeyValue.setContextValue(contextValue); - contextKeyValue.setContextKey(contextKey); - context.setContextKeyAndValue(contextKeyValue); - - // 3, get context - ContextValue contextValueResult = context.getContextValue(contextKey); - System.out.println("Got contextValue : " + new Gson().toJson(contextValueResult)); - System.out.println("Original contextValue : " + new Gson().toJson(contextValue)); - - contextClient.close(); - } catch (Exception e) { - if (null != contextClient) { - try { - contextClient.close(); - } catch (Exception e1) { - - } - } - e.printStackTrace(); - } - } -} diff --git a/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/RequestReadAllTask.java b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/RequestReadAllTask.java index a03f072c77..886b75ca65 100644 --- a/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/RequestReadAllTask.java +++ b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/RequestReadAllTask.java @@ -16,8 +16,6 @@ package com.webank.wedatasphere.linkis.protocol.query; -import com.webank.wedatasphere.linkis.protocol.task.Task; - /** * created by enjoyyin on 2018/10/9 * Description: diff --git a/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/ResponsePersist.java b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/ResponsePersist.java index f3a2b275e6..39dde90f0c 100644 --- a/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/ResponsePersist.java +++ b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/ResponsePersist.java @@ -29,7 +29,7 @@ import java.util.Map; public class ResponsePersist implements QueryProtocol { private Integer status; private String msg; - private Map<String,Object> data; + private Map<String, Object> data; public Integer getStatus() { return status; diff --git a/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/CacheNotFound.java b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/CacheNotFound.java new file mode 100644 index 0000000000..f0c719e689 --- /dev/null +++ b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/CacheNotFound.java @@ -0,0 +1,4 @@ +package com.webank.wedatasphere.linkis.protocol.query.cache; + +public class CacheNotFound implements ResponseReadCache { +} diff --git a/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/CacheTaskResult.java b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/CacheTaskResult.java new file mode 100644 index 0000000000..7f36f7d3b5 --- /dev/null +++ b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/CacheTaskResult.java @@ -0,0 +1,14 @@ +package com.webank.wedatasphere.linkis.protocol.query.cache; + +public class CacheTaskResult implements ResponseReadCache { + + private String resultLocation; + + public CacheTaskResult(String resultLocation) { + this.resultLocation = resultLocation; + } + + public String getResultLocation() { + return resultLocation; + } +} diff --git a/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/FailedToDeleteCache.java b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/FailedToDeleteCache.java new file mode 100644 index 0000000000..41cdca6ec2 --- /dev/null +++ b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/FailedToDeleteCache.java @@ -0,0 +1,13 @@ +package com.webank.wedatasphere.linkis.protocol.query.cache; + +public class FailedToDeleteCache { + private String errorMessage; + + public FailedToDeleteCache(String errorMessage) { + this.errorMessage = errorMessage; + } + + public String getErrorMessage() { + return errorMessage; + } +} diff --git a/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/RequestDeleteCache.java b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/RequestDeleteCache.java new file mode 100644 index 0000000000..317eed73ce --- /dev/null +++ b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/RequestDeleteCache.java @@ -0,0 +1,28 @@ +package com.webank.wedatasphere.linkis.protocol.query.cache; + +import com.webank.wedatasphere.linkis.protocol.query.QueryProtocol; + +public class RequestDeleteCache implements QueryProtocol { + + private String executionCode; + private String engineType; + private String user; + + public RequestDeleteCache(String executionCode, String engineType, String user) { + this.executionCode = executionCode; + this.engineType = engineType; + this.user = user; + } + + public String getExecutionCode() { + return executionCode; + } + + public String getEngineType() { + return engineType; + } + + public String getUser() { + return user; + } +} diff --git a/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/RequestReadCache.java b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/RequestReadCache.java new file mode 100644 index 0000000000..49ed99e2e2 --- /dev/null +++ b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/RequestReadCache.java @@ -0,0 +1,33 @@ +package com.webank.wedatasphere.linkis.protocol.query.cache; + +import com.webank.wedatasphere.linkis.protocol.query.QueryProtocol; + +public class RequestReadCache implements QueryProtocol { + private String executionCode; + private String engineType; + private String user; + private Long readCacheBefore; + + public RequestReadCache(String executionCode, String engineType, String user, Long readCacheBefore) { + this.executionCode = executionCode; + this.engineType = engineType; + this.user = user; + this.readCacheBefore = readCacheBefore; + } + + public String getExecutionCode() { + return executionCode; + } + + public String getEngineType() { + return engineType; + } + + public String getUser() { + return user; + } + + public Long getReadCacheBefore() { + return readCacheBefore; + } +} diff --git a/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/ResponseDeleteCache.java b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/ResponseDeleteCache.java new file mode 100644 index 0000000000..e9c0e3a9e3 --- /dev/null +++ b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/ResponseDeleteCache.java @@ -0,0 +1,6 @@ +package com.webank.wedatasphere.linkis.protocol.query.cache; + +import com.webank.wedatasphere.linkis.protocol.query.QueryProtocol; + +public interface ResponseDeleteCache extends QueryProtocol { +} diff --git a/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/ResponseReadCache.java b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/ResponseReadCache.java new file mode 100644 index 0000000000..dbcc84a50e --- /dev/null +++ b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/ResponseReadCache.java @@ -0,0 +1,6 @@ +package com.webank.wedatasphere.linkis.protocol.query.cache; + +import com.webank.wedatasphere.linkis.protocol.query.QueryProtocol; + +public interface ResponseReadCache extends QueryProtocol { +} diff --git a/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/SuccessDeletedCache.java b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/SuccessDeletedCache.java new file mode 100644 index 0000000000..b16098faad --- /dev/null +++ b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/SuccessDeletedCache.java @@ -0,0 +1,4 @@ +package com.webank.wedatasphere.linkis.protocol.query.cache; + +public class SuccessDeletedCache { +} diff --git a/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/usercontrol/RequestRegister.java b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/usercontrol/RequestRegister.java index 52bde811bb..f36a0211e2 100644 --- a/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/usercontrol/RequestRegister.java +++ b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/usercontrol/RequestRegister.java @@ -16,8 +16,6 @@ package com.webank.wedatasphere.linkis.protocol.usercontrol; -import java.util.Date; - /** * Created by alexyang */ diff --git a/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/usercontrol/ResponseRegister.java b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/usercontrol/ResponseRegister.java index b2b4217b09..5c12d62626 100644 --- a/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/usercontrol/ResponseRegister.java +++ b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/usercontrol/ResponseRegister.java @@ -21,7 +21,7 @@ import java.util.HashMap; /** * Created by alexyang */ -public class ResponseRegister implements UserControlRegtisterProtocol{ +public class ResponseRegister implements UserControlRegtisterProtocol { private int status; private String message; diff --git a/db/linkis_ddl.sql b/db/linkis_ddl.sql index 24d4bebb23..cf588c9424 100644 --- a/db/linkis_ddl.sql +++ b/db/linkis_ddl.sql @@ -642,3 +642,77 @@ CREATE TABLE `linkis_cs_context_listener` ( + +-- ---------------------------- +-- Table structure for linkis_datasource +-- ---------------------------- +CREATE TABLE IF NOT EXISTS `linkis_datasource` ( + `id` BIGINT(20) NOT NULL AUTO_INCREMENT, + `datasource_name` VARCHAR(100) NOT NULL COMMENT 'Data source name', + `datasource_type_id` BIGINT(20) DEFAULT NULL COMMENT 'Data source type id', + `datasource_desc` VARCHAR(200) DEFAULT NULL COMMENT 'Data source description', + `create_identify` VARCHAR(20) DEFAULT 'BDP' COMMENT 'Example: project name', + `create_system` VARCHAR(20) DEFAULT 'BDP' COMMENT 'Create system', + `create_user` VARCHAR(50) DEFAULT NULL COMMENT 'Creator', + `parameter` TEXT COMMENT 'Connect parameters', + `create_time` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `modify_user` VARCHAR(50) DEFAULT NULL COMMENT 'Modify user', + `modify_time` DATETIME DEFAULT NULL COMMENT 'Modify time', + `datasource_env_id` BIGINT(20) DEFAULT NULL, + PRIMARY KEY (`id`) + ) ENGINE=InnoDB AUTO_INCREMENT=140 DEFAULT CHARSET=utf8; + +-- ---------------------------- +-- Table structure for linkis_datasource_env +-- ---------------------------- +CREATE TABLE IF NOT EXISTS `linkis_datasource_env` ( + `id` BIGINT(20) NOT NULL AUTO_INCREMENT, + `env_name` VARCHAR(100) NOT NULL COMMENT 'Environment name', + `env_desc` VARCHAR(200) DEFAULT NULL COMMENT 'Description', + `create_user` VARCHAR(50) DEFAULT NULL COMMENT 'Creator', + `parameter` TEXT NOT NULL COMMENT 'Connect parameters', + `create_time` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `modify_user` VARCHAR(50) DEFAULT NULL COMMENT 'Modify user', + `modify_time` DATETIME DEFAULT NULL COMMENT 'Modify time', + PRIMARY KEY (`id`) + ) ENGINE=InnoDB AUTO_INCREMENT=108 DEFAULT CHARSET=utf8; + +-- ---------------------------- +-- Table structure for linkis_datasource_type_key +-- ---------------------------- +CREATE TABLE IF NOT EXISTS `linkis_datasource_type_key` ( + `id` BIGINT(20) NOT NULL AUTO_INCREMENT, + `key` VARCHAR(50) DEFAULT NULL COMMENT 'Key of variable', + `description` VARCHAR(200) DEFAULT NULL COMMENT 'Description', + `name` VARCHAR(50) DEFAULT NULL COMMENT 'Option name of column in page', + `data_source_type_id` BIGINT(20) DEFAULT NULL COMMENT 'Type id', + `require` TINYINT(1) DEFAULT '0', + `scope` VARCHAR(50) DEFAULT NULL COMMENT 'Scope', + `default_value` VARCHAR(200) DEFAULT NULL COMMENT 'Default value', + `value_type` VARCHAR(50) DEFAULT NULL COMMENT 'Value type', + `value_regex` VARCHAR(100) DEFAULT NULL COMMENT 'Value regex', + `ref_id` BIGINT(20) DEFAULT NULL COMMENT 'Related id', + `ref_value` VARCHAR(100) DEFAULT NULL COMMENT 'Related value', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- ---------------------------- +-- Table structure for linkis_datasource_type +-- ---------------------------- +CREATE TABLE IF NOT EXISTS `linkis_datasource_type` ( + `id` BIGINT(20) NOT NULL AUTO_INCREMENT, + `icon` VARCHAR(50) DEFAULT NULL COMMENT 'Icon', + `description` VARCHAR(200) DEFAULT NULL COMMENT 'Description', + `name` VARCHAR(50) DEFAULT NULL COMMENT 'Name', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- ---------------------------- +-- Table structure for linkis_datasource_type_env +-- ---------------------------- +CREATE TABLE IF NOT EXISTS `linkis_datasource_type_env` ( + `id` BIGINT(20) NOT NULL AUTO_INCREMENT, + `data_source_type_id` BIGINT(20) DEFAULT NULL COMMENT 'Type id', + `env_id` BIGINT(20) DEFAULT NULL COMMENT 'Environment id', + PRIMARY KEY (`id`) +)ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; \ No newline at end of file diff --git a/db/linkis_dml.sql b/db/linkis_dml.sql index b3edd58949..cec6d5495a 100644 --- a/db/linkis_dml.sql +++ b/db/linkis_dml.sql @@ -233,4 +233,8 @@ SELECT @tree_id1:=t.id from linkis_config_tree t LEFT JOIN linkis_application a insert into `linkis_config_key_tree` VALUES(NULL,@key_id1,@tree_id1); insert into `linkis_config_key_tree` VALUES(NULL,@key_id2,@tree_id1); -insert into `linkis_config_key_tree` VALUES(NULL,@key_id3,@tree_id1); \ No newline at end of file +insert into `linkis_config_key_tree` VALUES(NULL,@key_id3,@tree_id1); + +INSERT INTO `linkis_datasource_type`(`icon`, `name`) VALUES('0x001', 'ElasticSearch'); +INSERT INTO `linkis_datasource_type`(`icon`, `name`) VALUES('0x001', 'Hive'); +INSERT INTO `linkis_datasource_type`(`icon`, `name`) VALUES('0x001', 'MySql'); \ No newline at end of file diff --git a/db/module/linkis-datasource.sql b/db/module/linkis-datasource.sql new file mode 100644 index 0000000000..40a7de947c --- /dev/null +++ b/db/module/linkis-datasource.sql @@ -0,0 +1,67 @@ +CREATE TABLE IF NOT EXISTS `linkis_datasource` ( + `id` BIGINT(20) NOT NULL AUTO_INCREMENT, + `datasource_name` VARCHAR(100) NOT NULL COMMENT 'Data source name', + `datasource_type_id` BIGINT(20) DEFAULT NULL COMMENT 'Data source type id', + `datasource_desc` VARCHAR(200) DEFAULT NULL COMMENT 'Data source description', + `create_identify` VARCHAR(20) DEFAULT 'BDP' COMMENT 'Example: project name', + `create_system` VARCHAR(20) DEFAULT 'BDP' COMMENT 'Create system', + `create_user` VARCHAR(50) DEFAULT NULL COMMENT 'Creator', + `parameter` TEXT COMMENT 'Connect parameters', + `create_time` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `modify_user` VARCHAR(50) DEFAULT NULL COMMENT 'Modify user', + `modify_time` DATETIME DEFAULT NULL COMMENT 'Modify time', + `datasource_env_id` BIGINT(20) DEFAULT NULL, + PRIMARY KEY (`id`) + ) ENGINE=InnoDB AUTO_INCREMENT=140 DEFAULT CHARSET=utf8; + + +CREATE TABLE IF NOT EXISTS `linkis_datasource_env` ( + `id` BIGINT(20) NOT NULL AUTO_INCREMENT, + `env_name` VARCHAR(100) NOT NULL COMMENT 'Environment name', + `env_desc` VARCHAR(200) DEFAULT NULL COMMENT 'Description', + `create_user` VARCHAR(50) DEFAULT NULL COMMENT 'Creator', + `parameter` TEXT NOT NULL COMMENT 'Connect parameters', + `create_time` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `modify_user` VARCHAR(50) DEFAULT NULL COMMENT 'Modify user', + `modify_time` DATETIME DEFAULT NULL COMMENT 'Modify time', + PRIMARY KEY (`id`) + ) ENGINE=InnoDB AUTO_INCREMENT=108 DEFAULT CHARSET=utf8; + + +CREATE TABLE IF NOT EXISTS `linkis_datasource_type_key` ( + `id` BIGINT(20) NOT NULL AUTO_INCREMENT, + `key` VARCHAR(50) DEFAULT NULL COMMENT 'Key of variable', + `description` VARCHAR(200) DEFAULT NULL COMMENT 'Description', + `name` VARCHAR(50) DEFAULT NULL COMMENT 'Option name of column in page', + `data_source_type_id` BIGINT(20) DEFAULT NULL COMMENT 'Type id', + `require` TINYINT(1) DEFAULT '0', + `scope` VARCHAR(50) DEFAULT NULL COMMENT 'Scope', + `default_value` VARCHAR(200) DEFAULT NULL COMMENT 'Default value', + `value_type` VARCHAR(50) DEFAULT NULL COMMENT 'Value type', + `value_regex` VARCHAR(100) DEFAULT NULL COMMENT 'Value regex', + `ref_id` BIGINT(20) DEFAULT NULL COMMENT 'Related id', + `ref_value` VARCHAR(100) DEFAULT NULL COMMENT 'Related value', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + + +CREATE TABLE IF NOT EXISTS `linkis_datasource_type` ( + `id` BIGINT(20) NOT NULL AUTO_INCREMENT, + `icon` VARCHAR(50) DEFAULT NULL COMMENT 'Icon', + `description` VARCHAR(200) DEFAULT NULL COMMENT 'Description', + `name` VARCHAR(50) DEFAULT NULL COMMENT 'Name', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + + +CREATE TABLE IF NOT EXISTS `linkis_datasource_type_env` ( + `id` BIGINT(20) NOT NULL AUTO_INCREMENT, + `data_source_type_id` BIGINT(20) DEFAULT NULL COMMENT 'Type id', + `env_id` BIGINT(20) DEFAULT NULL COMMENT 'Environment id', + PRIMARY KEY (`id`) +)ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + + +INSERT INTO `linkis_datasource_type`(`icon`, `name`) VALUES('0x001', 'ElasticSearch'); +INSERT INTO `linkis_datasource_type`(`icon`, `name`) VALUES('0x001', 'Hive'); +INSERT INTO `linkis_datasource_type`(`icon`, `name`) VALUES('0x001', 'MySql'); \ No newline at end of file diff --git a/metadata/bin/start-metadata.sh b/metadata/bin/start-metadata.sh index 56cab6df00..80cc775a4a 100644 --- a/metadata/bin/start-metadata.sh +++ b/metadata/bin/start-metadata.sh @@ -26,7 +26,7 @@ if [[ -f "${SERVER_PID}" ]]; then fi fi -nohup java $SERVER_JAVA_OPTS -cp $HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out & +nohup java $SERVER_JAVA_OPTS -cp ../module/lib/*:$HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out & pid=$! if [[ -z "${pid}" ]]; then echo "server $SERVER_NAME start failed!" diff --git a/pom.xml b/pom.xml index db47d54209..06a0f64d50 100644 --- a/pom.xml +++ b/pom.xml @@ -57,42 +57,41 @@ <modules> - <module>core/common</module> + <module>core/common</module> <module>core/cloudProtocol</module> <module>core/scheduler</module> <module>core/cloudModule</module> <module>core/cloudRPC</module> <module>core/cloudMybatis</module> - <module>core/httpclient</module> <module>core/hadoop-common</module> + <module>core/httpclient</module> <module>storage/storage</module> - <module>bml/bmlcommon</module> <module>bml/bmlclient</module> + <module>bml/bmlserver</module> <module>resourceManager/resourcemanagercommon</module> <module>resourceManager/resourcemanagerclient</module> - <module>gateway/core</module> - <module>gateway/springcloudgateway</module> - <module>gateway/gateway-httpclient-support</module> - + <module>resourceManager/resourcemanagerserver</module> <module>publicService/udf</module> <module>publicService/application</module> <module>publicService/jobhistory</module> <module>publicService/configuration</module> <module>publicService/variable</module> + <module>contextservice/cs-common</module> + <module>contextservice/cs-client</module> + <module>contextservice/cs-listener</module> + <module>contextservice/cs-ujes-client</module> + <module>contextservice/cs-cache</module> + <module>contextservice/cs-persistence</module> + <module>contextservice/cs-highavailable</module> + <module>ujes/entrance</module> + <module>ujes/entranceclient</module> <module>publicService/workspace</module> <module>publicService/workspace/client/workspace-httpclient</module> - <module>bml/bmlserver</module> - <module>resourceManager/resourcemanagerserver</module> <module>metadata</module> - <module>gateway/gateway-ujes-support</module> - <module>eurekaServer</module> - <module>publicService</module> - <module>userControl</module> <module>ujes/engine</module> - <module>ujes/enginemanager</module> - <module>ujes/entrance</module> <module>bml/bml-engine-hook</module> + <module>ujes/enginemanager</module> <module>ujes/definedEngines/spark/engine</module> <module>ujes/definedEngines/spark/enginemanager</module> <module>ujes/definedEngines/spark/entrance</module> @@ -101,26 +100,37 @@ <module>ujes/definedEngines/hive/enginemanager</module> <module>ujes/definedEngines/hive/entrance</module> - <module>datasource/datasourcemanager/common</module> - <module>datasource/datasourcemanager/server</module> - <module>datasource/metadatamanager/common</module> - <module>datasource/metadatamanager/server</module> - <module>datasource/metadatamanager/service/elasticsearch</module> - <module>datasource/metadatamanager/service/hive</module> - <module>datasource/metadatamanager/service/mysql</module> - <module>ujes/definedEngines/python/engine</module> <module>ujes/definedEngines/python/enginemanager</module> <module>ujes/definedEngines/python/entrance</module> - <module>ujes/definedEngines/shell/engine</module> - <module>ujes/definedEngines/shell/enginemanager</module> - <module>ujes/definedEngines/shell/entrance</module> - <module>ujes/definedEngines/jdbc/entrance</module> - <module>ujes/client</module> + <module>ujes/definedEngines/pipeline/engine</module> <module>ujes/definedEngines/pipeline/enginemanager</module> <module>ujes/definedEngines/pipeline/entrance</module> + <module>ujes/definedEngines/jdbc/entrance</module> <module>ujes/definedEngines/mlsql/entrance</module> + + <module>ujes/definedEngines/shell/engine</module> + <module>ujes/definedEngines/shell/entrance</module> + <module>ujes/definedEngines/shell/enginemanager</module> + + + <module>gateway/core</module> + <module>gateway/springcloudgateway</module> + <module>gateway/gateway-ujes-support</module> + <module>gateway/gateway-httpclient-support</module> + <module>eurekaServer</module> + <module>publicService</module> + <module>ujes/client</module> + <module>contextservice/cs-server</module> + <module>contextservice/cs-search</module> + <module>datasource/metadatamanager/common</module> + <module>datasource/metadatamanager/server</module> + <module>datasource/metadatamanager/service/mysql</module> + <module>datasource/metadatamanager/service/elasticsearch</module> + <module>datasource/metadatamanager/service/hive</module> + <module>datasource/datasourcemanager/common</module> + <module>datasource/datasourcemanager/server</module> <module>assembly/public-module</module> <module>assembly</module> @@ -132,7 +142,7 @@ <spring.eureka.version>1.4.4.RELEASE</spring.eureka.version> <spring.boot.version>2.0.3.RELEASE</spring.boot.version> <gson.version>2.8.5</gson.version> - <fasterxml.jackson.version>2.9.6</fasterxml.jackson.version> + <fasterxml.jackson.version>2.10.0</fasterxml.jackson.version> <scala.version>2.11.8</scala.version> <jdk.compile.version>1.8</jdk.compile.version> <plugin.scala.version>2.15.2</plugin.scala.version> @@ -141,12 +151,12 @@ <jersey.servlet.version>2.23.1</jersey.servlet.version> <jetty.version>9.4.11.v20180605</jetty.version> <httpclient.version>4.5.4</httpclient.version> + <httpmime.version>4.5.4</httpmime.version> <slf4j.version>1.7.12</slf4j.version> <linkis.version>0.9.4</linkis.version> <maven.version>3.3.3</maven.version> + <xstream.core.version>1.4.11.1</xstream.core.version> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - <dispatch.version>0.11.2</dispatch.version> - <jsoup.version>1.8.2</jsoup.version> </properties> <dependencyManagement> @@ -192,6 +202,11 @@ <artifactId>gson</artifactId> <version>${gson.version}</version> </dependency> + <dependency> + <groupId>com.thoughtworks.xstream</groupId> + <artifactId>xstream</artifactId> + <version>${xstream.core.version}</version> + </dependency> <!-- scala version --> <dependency> @@ -222,7 +237,7 @@ <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> - <version>14.0</version> + <version>25.1-jre</version> </dependency> <dependency> <groupId>org.slf4j</groupId> @@ -272,44 +287,44 @@ <id>release</id> <build> <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-source-plugin</artifactId> - <version>3.1.0</version> - <configuration> - <attach>true</attach> - </configuration> - <executions> - <execution> - <id>create-source-jar</id> - <goals> - <goal>jar-no-fork</goal> - <goal>test-jar-no-fork</goal> - </goals> - </execution> - </executions> - </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-source-plugin</artifactId> + <version>3.1.0</version> + <configuration> + <attach>true</attach> + </configuration> + <executions> + <execution> + <id>create-source-jar</id> + <goals> + <goal>jar-no-fork</goal> + <goal>test-jar-no-fork</goal> + </goals> + </execution> + </executions> + </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-deploy-plugin</artifactId> - <version>3.0.0-M1</version> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-javadoc-plugin</artifactId> - <version>3.0.1</version> - <executions> - <execution> - <id>attach-javadocs</id> - <goals> - <goal>jar</goal> - </goals> - </execution> - </executions> - </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-deploy-plugin</artifactId> + <version>3.0.0-M1</version> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-javadoc-plugin</artifactId> + <version>3.0.1</version> + <executions> + <execution> + <id>attach-javadocs</id> + <goals> + <goal>jar</goal> + </goals> + </execution> + </executions> + </plugin> - </plugins> + </plugins> </build> </profile> diff --git a/publicService/bin/start-publicservice.sh b/publicService/bin/start-publicservice.sh index 56cab6df00..80cc775a4a 100644 --- a/publicService/bin/start-publicservice.sh +++ b/publicService/bin/start-publicservice.sh @@ -26,7 +26,7 @@ if [[ -f "${SERVER_PID}" ]]; then fi fi -nohup java $SERVER_JAVA_OPTS -cp $HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out & +nohup java $SERVER_JAVA_OPTS -cp ../module/lib/*:$HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out & pid=$! if [[ -z "${pid}" ]]; then echo "server $SERVER_NAME start failed!" diff --git a/resourceManager/resourcemanagerserver/src/test/scala/com/webank/bdp/dataworkcloud/resourcemanager/notify/NotifyRMEventSerializerTest.scala b/resourceManager/resourcemanagerserver/src/test/scala/com/webank/bdp/dataworkcloud/resourcemanager/notify/NotifyRMEventSerializerTest.scala deleted file mode 100644 index 7d552aaf62..0000000000 --- a/resourceManager/resourcemanagerserver/src/test/scala/com/webank/bdp/dataworkcloud/resourcemanager/notify/NotifyRMEventSerializerTest.scala +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.webank.bdp.dataworkcloud.resourcemanager.notify - -import com.webank.wedatasphere.linkis.resourcemanager.event.notify._ -import org.json4s.DefaultFormats -import org.json4s.jackson.Serialization.{read, write} - -object NotifyRMEventSerializerTest { - - implicit val formats = DefaultFormats + NotifyRMEventSerializer - var event: NotifyRMEvent = _ - var serialized: String = _ - - def main(args: Array[String]): Unit = { - - } - - private def checkResult = { - serialized = write(event) - System.out.println(serialized) - val recovered = read[NotifyRMEvent](serialized) - System.out.println(write(recovered)) - } -} diff --git a/resourceManager/resourcemanagerserver/src/test/scala/com/webank/bdp/dataworkcloud/resourcemanager/notify/ResourceTest.scala b/resourceManager/resourcemanagerserver/src/test/scala/com/webank/bdp/dataworkcloud/resourcemanager/notify/ResourceTest.scala deleted file mode 100644 index 4de65d9cf3..0000000000 --- a/resourceManager/resourcemanagerserver/src/test/scala/com/webank/bdp/dataworkcloud/resourcemanager/notify/ResourceTest.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.webank.bdp.dataworkcloud.resourcemanager.notify - -object ResourceTest { - def main(args: Array[String]): Unit = { - - } -} diff --git a/resourceManager/resourcemanagerserver/src/test/scala/com/webank/bdp/dataworkcloud/resourcemanager/notify/ZookeeperLockTest.scala b/resourceManager/resourcemanagerserver/src/test/scala/com/webank/bdp/dataworkcloud/resourcemanager/notify/ZookeeperLockTest.scala deleted file mode 100644 index 6a16cbeba4..0000000000 --- a/resourceManager/resourcemanagerserver/src/test/scala/com/webank/bdp/dataworkcloud/resourcemanager/notify/ZookeeperLockTest.scala +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.webank.bdp.dataworkcloud.resourcemanager.notify - -object ZookeeperLockTest { - - def main(args: Array[String]): Unit = { - - val thread1 = new TestThread("a") - val thread2 = new TestThread("b") - val thread3 = new TestThread("c") - val thread4 = new TestThread("d") - val thread5 = new TestThread("e") - - thread1.start() - thread2.start() - thread3.start() - thread4.start() - thread5.start() - - } - - class TestThread(name: String) extends Thread { - override def run(): Unit = { - val lock = ZookeeperDistributedLock("/dwc_test", "test_lock") - try { - lock.acquire() - System.out.println(name + " get lock") - Thread.sleep(3000) - } finally { - lock.release() - System.out.println(name + " release lock") - } - } - } - -} diff --git a/resourceManager/resourcemanagerserver/src/test/scala/com/webank/bdp/dataworkcloud/resourcemanager/notify/ZookeeperPubTest.scala b/resourceManager/resourcemanagerserver/src/test/scala/com/webank/bdp/dataworkcloud/resourcemanager/notify/ZookeeperPubTest.scala deleted file mode 100644 index 6f02776c86..0000000000 --- a/resourceManager/resourcemanagerserver/src/test/scala/com/webank/bdp/dataworkcloud/resourcemanager/notify/ZookeeperPubTest.scala +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.webank.bdp.dataworkcloud.resourcemanager.notify - -import com.webank.wedatasphere.linkis.resourcemanager.event.EventScope -import com.webank.wedatasphere.linkis.resourcemanager.event.notify.DefaultNotifyRMEvent - -object ZookeeperPubTest { - def main(args: Array[String]): Unit = { - val threadPub = new PubThread - threadPub.start() - } - -} - -class PubThread extends Thread { - val modules = Array("a", "b", "c", "d", "e", "f") - val scopes = Array(EventScope.Instance, EventScope.Other, EventScope.Service, EventScope.User) - val publisher = NotifyRMEventPublisher("test_queue") - - override def run(): Unit = { - (0 to 100).foreach { i => - publisher.publish(new DefaultNotifyRMEvent(i.toString, modules(i % modules.length), scopes(i % scopes.length))) - Thread.sleep(500) - } - } -} \ No newline at end of file diff --git a/resourceManager/resourcemanagerserver/src/test/scala/com/webank/bdp/dataworkcloud/resourcemanager/notify/ZookeeperQueueTest.scala b/resourceManager/resourcemanagerserver/src/test/scala/com/webank/bdp/dataworkcloud/resourcemanager/notify/ZookeeperQueueTest.scala deleted file mode 100644 index b78e6307db..0000000000 --- a/resourceManager/resourcemanagerserver/src/test/scala/com/webank/bdp/dataworkcloud/resourcemanager/notify/ZookeeperQueueTest.scala +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.webank.bdp.dataworkcloud.resourcemanager.notify - -import com.webank.wedatasphere.linkis.resourcemanager.event.EventScope.{EventScope => _, _} -import com.webank.wedatasphere.linkis.resourcemanager.event.notify.DefaultNotifyRMEvent -import com.webank.wedatasphere.linkis.resourcemanager.schedule.{EventSchedulerContextImpl, EventSchedulerImpl} - -object ZookeeperQueueTest { - - def main(args: Array[String]): Unit = { - - val threadSub = new Thread { - val subscriber = NotifyRMEventSubscriber("test_queue", new EventSchedulerImpl(new EventSchedulerContextImpl(1))) - - override def run(): Unit = { - subscriber.start() - Thread.sleep(60000) - subscriber.stop() - } - } - - val threadSub1 = new Thread { - val subscriber = NotifyRMEventSubscriber("test_queue", new EventSchedulerImpl(new EventSchedulerContextImpl(1))) - - override def run(): Unit = { - subscriber.start() - Thread.sleep(60000) - subscriber.stop() - } - } - - val threadPub = new Thread { - val publisher = NotifyRMEventPublisher("test_queue") - - override def run(): Unit = { - (0 to 1).foreach { i => - publisher.publish(new DefaultNotifyRMEvent(i.toString, "test", Instance)) - } - - } - } - - threadSub1.start() - threadPub.start() - threadSub.start() - - } -} diff --git a/resourceManager/resourcemanagerserver/src/test/scala/com/webank/bdp/dataworkcloud/resourcemanager/notify/ZookeeperSubTest.scala b/resourceManager/resourcemanagerserver/src/test/scala/com/webank/bdp/dataworkcloud/resourcemanager/notify/ZookeeperSubTest.scala deleted file mode 100644 index 9957c8c875..0000000000 --- a/resourceManager/resourcemanagerserver/src/test/scala/com/webank/bdp/dataworkcloud/resourcemanager/notify/ZookeeperSubTest.scala +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.webank.bdp.dataworkcloud.resourcemanager.notify - -import com.webank.wedatasphere.linkis.resourcemanager.schedule.{EventSchedulerContextImpl, EventSchedulerImpl} - -object ZookeeperSubTest { - def main(args: Array[String]): Unit = { - val threadSub = new SubThread - val threadSub1 = new SubThread - - threadSub.start() - threadSub1.start() - } -} - -class SubThread extends Thread { - val subscriber = NotifyRMEventSubscriber("test_queue", new EventSchedulerImpl(new EventSchedulerContextImpl(1))) - - override def run(): Unit = { - subscriber.start() - Thread.sleep(60000) - subscriber.stop() - } -} diff --git a/resourceManager/resourcemanagerserver/src/test/scala/com/webank/test/TestMain.scala b/resourceManager/resourcemanagerserver/src/test/scala/com/webank/test/TestMain.scala deleted file mode 100644 index 5e392a66e7..0000000000 --- a/resourceManager/resourcemanagerserver/src/test/scala/com/webank/test/TestMain.scala +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.webank.test - -import com.webank.wedatasphere.linkis.resourcemanager.domain.{ModuleInfoSerializer, ModuleInstanceSerializer} -import com.webank.wedatasphere.linkis.resourcemanager.event.notify.NotifyRMEventSerializer -import com.webank.wedatasphere.linkis.resourcemanager.{ResourceSerializer, YarnResource} -import org.json4s.JsonAST.JObject -import org.json4s.JsonDSL._ -import org.json4s.{CustomSerializer, DefaultFormats, Extraction} - - -/** - * Created by johnnwang on 9/15/18. - */ -class ResultResource - -case class NotEnoughResource(reason: String = null) extends ResultResource - -class AvailableResource(val ticketId: String) extends ResultResource - -case class UserResultResource(ticketId: String, user: String) extends ResultResource - -object ResultResourceSerializer1 extends CustomSerializer[ResultResource](implicit formats => ( { - case JObject(List(("AvailableResource", JObject(List(("ticketId", ticketId)))))) => new AvailableResource(ticketId.extract[String]) -}, { - case r: AvailableResource => ("AvailableResource", ("ticketId", Extraction.decompose(r.ticketId))) -})) - -object ResultResourceSerializer2 extends CustomSerializer[ResultResource](implicit formats => ( { - case JObject(List(("NotEnoughResource", JObject(List(("reason", reason)))))) => NotEnoughResource(reason.extract[String]) - // case JObject(List(("ticketId", ticketId))) => new AvailableResource(ticketId.extract[String]) - case JObject(List(("UserResultResource", JObject(List(("ticketId", ticketId), ("user", user)))))) => - UserResultResource(ticketId.extract[String], user.extract[String]) -}, { - case r: NotEnoughResource => ("NotEnoughResource", ("reason", Extraction.decompose(r.reason))) - case r: AvailableResource => ("ticketId", Extraction.decompose(r.ticketId)) - case r: UserResultResource => ("UserResultResource", ("ticketId", r.ticketId) ~ ("user", r.user)) -})) - -object TestMain { - implicit val formats = DefaultFormats + ResultResourceSerializer1 + ResourceSerializer + ModuleInfoSerializer + ModuleInstanceSerializer + NotifyRMEventSerializer - - //implicit val formats = DefaultFormats + ResourceSerializer + ModuleResourceInfoSerializer + ResultResourceSerializer1 + ModuleInstanceSerializer + ModuleInfoSerializer - def main(args: Array[String]): Unit = { - val yarnResource1 = new YarnResource(100, 10, 0, "ide") - val yarnResource2 = new YarnResource(1, 1, 0, "ide") - println(yarnResource1 > yarnResource2) - } -} diff --git a/resourceManager/resourcemanagerserver/src/test/scala/com/webank/test/TestResource.scala b/resourceManager/resourcemanagerserver/src/test/scala/com/webank/test/TestResource.scala deleted file mode 100644 index a67287cb19..0000000000 --- a/resourceManager/resourcemanagerserver/src/test/scala/com/webank/test/TestResource.scala +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.webank.test - -import com.webank.wedatasphere.linkis.resourcemanager.utils.YarnUtil - -/** - * Created by johnnwang on 2018/12/29. - */ -object TestResource { - - def main(args: Array[String]): Unit = { - val info = YarnUtil.getApplicationsInfo("q01") - print(info) - } - - // def main(args: Array[String]): Unit = { - // val resource1 = new DriverAndYarnResource(new LoadInstanceResource(100,100,0), - // new YarnResource(101440 * 1024l * 1024l,50,2,"q05")) - // val resource2 = new DriverAndYarnResource(new LoadInstanceResource(50,50,0), - // new YarnResource(103424 * 1024l * 1024l,2,0,"q05")) - // - // println((resource1 - resource2).toString) - // } - // def minus(resource1:Resource,resource2:Resource) = { - // resource1 - resource2 - // } -} diff --git a/storage/storage/pom.xml b/storage/storage/pom.xml index f12d217ae1..221b845f15 100644 --- a/storage/storage/pom.xml +++ b/storage/storage/pom.xml @@ -52,6 +52,10 @@ <groupId>org.scala-lang</groupId> <artifactId>scala-library</artifactId> </exclusion> + <exclusion> + <artifactId>jackson-databind</artifactId> + <groupId>com.fasterxml.jackson.core</groupId> + </exclusion> </exclusions> <scope>provided</scope> </dependency> @@ -70,6 +74,12 @@ <groupId>com.monitorjbl</groupId> <artifactId>xlsx-streamer</artifactId> <version>1.2.1</version> + <exclusions> + <exclusion> + <artifactId>jackson-databind</artifactId> + <groupId>com.fasterxml.jackson.core</groupId> + </exclusion> + </exclusions> </dependency> </dependencies> diff --git a/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/factory/BuildFactory.java b/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/factory/BuildFactory.java index 2351ad2752..a207bc8841 100644 --- a/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/factory/BuildFactory.java +++ b/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/factory/BuildFactory.java @@ -17,7 +17,6 @@ package com.webank.wedatasphere.linkis.storage.factory; import com.webank.wedatasphere.linkis.common.io.Fs; -import com.webank.wedatasphere.linkis.common.io.FsPath; /** * Created by johnnwang on 10/17/18. diff --git a/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/factory/impl/BuildLocalFileSystem.java b/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/factory/impl/BuildLocalFileSystem.java index 2b22b41d5f..d31d235241 100644 --- a/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/factory/impl/BuildLocalFileSystem.java +++ b/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/factory/impl/BuildLocalFileSystem.java @@ -17,7 +17,6 @@ package com.webank.wedatasphere.linkis.storage.factory.impl; import com.webank.wedatasphere.linkis.common.io.Fs; -import com.webank.wedatasphere.linkis.common.io.FsPath; import com.webank.wedatasphere.linkis.storage.factory.BuildFactory; import com.webank.wedatasphere.linkis.storage.fs.FileSystem; import com.webank.wedatasphere.linkis.storage.fs.impl.LocalFileSystem; diff --git a/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/fs/FileSystem.java b/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/fs/FileSystem.java index de51915d52..f361006425 100644 --- a/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/fs/FileSystem.java +++ b/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/fs/FileSystem.java @@ -22,7 +22,6 @@ import com.webank.wedatasphere.linkis.storage.domain.FsPathListWithError; import java.io.File; import java.io.IOException; -import java.util.Map; /** * Created by johnnwang on 10/15/18. diff --git a/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/fs/impl/LocalFileSystem.java b/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/fs/impl/LocalFileSystem.java index d14ab084a0..b7a00606c2 100644 --- a/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/fs/impl/LocalFileSystem.java +++ b/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/fs/impl/LocalFileSystem.java @@ -17,8 +17,6 @@ package com.webank.wedatasphere.linkis.storage.fs.impl; import com.webank.wedatasphere.linkis.common.io.FsPath; -import com.webank.wedatasphere.linkis.common.io.FsReader; -import com.webank.wedatasphere.linkis.common.io.FsWriter; import com.webank.wedatasphere.linkis.common.utils.Utils; import com.webank.wedatasphere.linkis.storage.domain.FsPathListWithError; import com.webank.wedatasphere.linkis.storage.exception.StorageWarnException; diff --git a/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/pipeline/PipelineReader.java b/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/pipeline/PipelineReader.java index 60730049ff..323a1d8403 100644 --- a/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/pipeline/PipelineReader.java +++ b/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/pipeline/PipelineReader.java @@ -19,7 +19,6 @@ package com.webank.wedatasphere.linkis.storage.pipeline; import com.webank.wedatasphere.linkis.common.io.FsReader; import java.util.Iterator; -import java.util.List; import java.util.Map; /** diff --git a/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/pipeline/PipelineWriter.java b/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/pipeline/PipelineWriter.java index ababb0d593..a6e9d1a993 100644 --- a/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/pipeline/PipelineWriter.java +++ b/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/pipeline/PipelineWriter.java @@ -18,7 +18,6 @@ package com.webank.wedatasphere.linkis.storage.pipeline; import com.webank.wedatasphere.linkis.common.io.FsWriter; -import java.io.IOException; import java.util.Map; /** diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/FSFactory.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/FSFactory.scala index cc9868b924..6b7aa39bfb 100644 --- a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/FSFactory.scala +++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/FSFactory.scala @@ -21,7 +21,6 @@ import com.webank.wedatasphere.linkis.common.utils.Logging import com.webank.wedatasphere.linkis.storage.exception.StorageFatalException import com.webank.wedatasphere.linkis.storage.factory.BuildFactory import com.webank.wedatasphere.linkis.storage.utils.{StorageConfiguration, StorageUtils} -import net.sf.cglib.proxy.Enhancer /** * Created by johnnwang on 10/17/18. diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/domain/DataType.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/domain/DataType.scala index eb6c32b3eb..01153e4b72 100644 --- a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/domain/DataType.scala +++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/domain/DataType.scala @@ -17,8 +17,8 @@ package com.webank.wedatasphere.linkis.storage.domain import java.sql.{Date, Timestamp} -import com.webank.wedatasphere.linkis.common.utils.Utils -import com.webank.wedatasphere.linkis.common.utils.Logging + +import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils} /** * Created by johnnwang on 10/17/18. diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/DefaultResultSetFactory.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/DefaultResultSetFactory.scala index 5697b3ddb3..618da1bc16 100644 --- a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/DefaultResultSetFactory.scala +++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/DefaultResultSetFactory.scala @@ -18,12 +18,12 @@ package com.webank.wedatasphere.linkis.storage.resultset import java.util -import com.webank.wedatasphere.linkis.common.io.{FsPath, MetaData, Record} import com.webank.wedatasphere.linkis.common.io.resultset.ResultSet +import com.webank.wedatasphere.linkis.common.io.{FsPath, MetaData, Record} import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils} import com.webank.wedatasphere.linkis.storage.FSFactory import com.webank.wedatasphere.linkis.storage.domain.Dolphin -import com.webank.wedatasphere.linkis.storage.exception.{StorageErrorException, StorageFatalException, StorageWarnException} +import com.webank.wedatasphere.linkis.storage.exception.{StorageErrorException, StorageWarnException} import com.webank.wedatasphere.linkis.storage.utils.{StorageConfiguration, StorageUtils} import org.apache.commons.lang.StringUtils @@ -75,6 +75,7 @@ class DefaultResultSetFactory extends ResultSetFactory with Logging{ val resultSetType = Dolphin.getType(inputStream) if(StringUtils.isEmpty(resultSetType)) throw new StorageWarnException(51000, s"The file (${fsPath.getPath}) is empty(文件(${fsPath.getPath}) 为空)") Utils.tryQuietly(inputStream.close()) + Utils.tryQuietly(fs.close()) getResultSetByType(resultSetType) } diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/StorageResultSet.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/StorageResultSet.scala index 511ab9db85..ca8f708df5 100644 --- a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/StorageResultSet.scala +++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/StorageResultSet.scala @@ -16,8 +16,8 @@ package com.webank.wedatasphere.linkis.storage.resultset +import com.webank.wedatasphere.linkis.common.io.resultset.ResultSet import com.webank.wedatasphere.linkis.common.io.{FsPath, MetaData, Record} -import com.webank.wedatasphere.linkis.common.io.resultset.{ResultDeserializer, ResultSerializer, ResultSet} import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils} import com.webank.wedatasphere.linkis.storage.domain.Dolphin import com.webank.wedatasphere.linkis.storage.utils.StorageConfiguration diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/StorageResultSetReader.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/StorageResultSetReader.scala index ee15588e91..e0fece3215 100644 --- a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/StorageResultSetReader.scala +++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/StorageResultSetReader.scala @@ -20,7 +20,7 @@ import java.io.{ByteArrayInputStream, IOException, InputStream} import com.webank.wedatasphere.linkis.common.io.resultset.{ResultSet, ResultSetReader} import com.webank.wedatasphere.linkis.common.io.{MetaData, Record} -import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils} +import com.webank.wedatasphere.linkis.common.utils.Logging import com.webank.wedatasphere.linkis.storage.domain.Dolphin import com.webank.wedatasphere.linkis.storage.exception.StorageWarnException import com.webank.wedatasphere.linkis.storage.utils.StorageUtils diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/StorageResultSetWriter.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/StorageResultSetWriter.scala index be6a4f172c..b5f9218539 100644 --- a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/StorageResultSetWriter.scala +++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/StorageResultSetWriter.scala @@ -17,14 +17,12 @@ package com.webank.wedatasphere.linkis.storage.resultset import java.io.{IOException, OutputStream} -import java.util import com.webank.wedatasphere.linkis.common.io.resultset.{ResultSerializer, ResultSet, ResultSetWriter} import com.webank.wedatasphere.linkis.common.io.{Fs, FsPath, MetaData, Record} import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils} import com.webank.wedatasphere.linkis.storage.FSFactory import com.webank.wedatasphere.linkis.storage.domain.Dolphin -import com.webank.wedatasphere.linkis.storage.exception.StorageErrorException import com.webank.wedatasphere.linkis.storage.utils.{FileSystemUtils, StorageUtils} import scala.collection.mutable.ArrayBuffer @@ -55,6 +53,8 @@ class StorageResultSetWriter[K <: MetaData, V <: Record](resultSet: ResultSet[K, private var proxyUser:String = StorageUtils.getJvmUser + def getMetaData: MetaData = rMetaData + def setProxyUser(proxyUser:String): Unit = { this.proxyUser = proxyUser } @@ -72,7 +72,7 @@ class StorageResultSetWriter[K <: MetaData, V <: Record](resultSet: ResultSet[K, fs = FSFactory.getFsByProxyUser(storePath,proxyUser) fs.init(null) FileSystemUtils.createNewFile(storePath, proxyUser,true) - outputStream = fs.write(storePath, false) + outputStream = fs.write(storePath, true) info(s"Succeed to create a new file:$storePath") } } diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/html/HtmlResultSet.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/html/HtmlResultSet.scala index 99a6d0e631..a2f16c73b7 100644 --- a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/html/HtmlResultSet.scala +++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/html/HtmlResultSet.scala @@ -18,8 +18,8 @@ package com.webank.wedatasphere.linkis.storage.resultset.html import com.webank.wedatasphere.linkis.common.io.resultset.{ResultDeserializer, ResultSerializer} import com.webank.wedatasphere.linkis.storage.resultset.txt.{TextResultDeserializer, TextResultSerializer} -import com.webank.wedatasphere.linkis.storage.{LineMetaData, LineRecord} import com.webank.wedatasphere.linkis.storage.resultset.{ResultSetFactory, StorageResultSet} +import com.webank.wedatasphere.linkis.storage.{LineMetaData, LineRecord} /** * Created by johnnwang on 2018/12/10. diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/io/IOResultSerializer.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/io/IOResultSerializer.scala index 84b9ce9e66..bc9ecd1fe5 100644 --- a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/io/IOResultSerializer.scala +++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/io/IOResultSerializer.scala @@ -16,8 +16,8 @@ package com.webank.wedatasphere.linkis.storage.resultset.io -import com.webank.wedatasphere.linkis.common.io.{MetaData, Record} import com.webank.wedatasphere.linkis.common.io.resultset.ResultSerializer +import com.webank.wedatasphere.linkis.common.io.{MetaData, Record} import com.webank.wedatasphere.linkis.storage.domain.Dolphin import org.apache.commons.codec.binary.Base64 diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/picture/PictureResultSet.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/picture/PictureResultSet.scala index 113d0e9e59..ce8c4ec1f1 100644 --- a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/picture/PictureResultSet.scala +++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/picture/PictureResultSet.scala @@ -17,9 +17,9 @@ package com.webank.wedatasphere.linkis.storage.resultset.picture import com.webank.wedatasphere.linkis.common.io.resultset.{ResultDeserializer, ResultSerializer} -import com.webank.wedatasphere.linkis.storage.{LineMetaData, LineRecord} -import com.webank.wedatasphere.linkis.storage.resultset.{ResultSetFactory, StorageResultSet} import com.webank.wedatasphere.linkis.storage.resultset.txt.{TextResultDeserializer, TextResultSerializer} +import com.webank.wedatasphere.linkis.storage.resultset.{ResultSetFactory, StorageResultSet} +import com.webank.wedatasphere.linkis.storage.{LineMetaData, LineRecord} /** * Created by johnnwang on 2018/12/10. diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/txt/TextResultDeserializer.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/txt/TextResultDeserializer.scala index 9018d967e0..63b2114aca 100644 --- a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/txt/TextResultDeserializer.scala +++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/txt/TextResultDeserializer.scala @@ -17,8 +17,8 @@ package com.webank.wedatasphere.linkis.storage.resultset.txt import com.webank.wedatasphere.linkis.common.io.resultset.ResultDeserializer -import com.webank.wedatasphere.linkis.storage.{LineMetaData, LineRecord} import com.webank.wedatasphere.linkis.storage.domain.Dolphin +import com.webank.wedatasphere.linkis.storage.{LineMetaData, LineRecord} /** * Created by johnnwang on 10/20/18. diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/txt/TextResultSerializer.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/txt/TextResultSerializer.scala index 232fce723f..940cf282b3 100644 --- a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/txt/TextResultSerializer.scala +++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/txt/TextResultSerializer.scala @@ -16,12 +16,10 @@ package com.webank.wedatasphere.linkis.storage.resultset.txt -import com.webank.wedatasphere.linkis.common.io.{MetaData, Record} import com.webank.wedatasphere.linkis.common.io.resultset.ResultSerializer -import com.webank.wedatasphere.linkis.storage.{LineMetaData, LineRecord} +import com.webank.wedatasphere.linkis.common.io.{MetaData, Record} import com.webank.wedatasphere.linkis.storage.domain.Dolphin - -import scala.tools.scalap.scalax.util.StringUtil +import com.webank.wedatasphere.linkis.storage.{LineMetaData, LineRecord} /** * Created by johnnwang on 10/20/18. diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/txt/TextResultSet.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/txt/TextResultSet.scala index 0a0bbc09cb..ae3b19bfc8 100644 --- a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/txt/TextResultSet.scala +++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/txt/TextResultSet.scala @@ -17,8 +17,8 @@ package com.webank.wedatasphere.linkis.storage.resultset.txt import com.webank.wedatasphere.linkis.common.io.resultset.{ResultDeserializer, ResultSerializer} -import com.webank.wedatasphere.linkis.storage.{LineMetaData, LineRecord} import com.webank.wedatasphere.linkis.storage.resultset.{ResultSetFactory, StorageResultSet} +import com.webank.wedatasphere.linkis.storage.{LineMetaData, LineRecord} /** * Created by johnnwang on 10/20/18. diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/script/VariableParser.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/script/VariableParser.scala index 2c8831430e..81bc6a0a2c 100644 --- a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/script/VariableParser.scala +++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/script/VariableParser.scala @@ -16,12 +16,8 @@ package com.webank.wedatasphere.linkis.storage.script -import java.io.{FileInputStream, FileOutputStream} import java.util -import com.google.gson.{Gson, GsonBuilder} -import com.webank.wedatasphere.linkis.common.io.FsPath - import scala.collection.mutable.ArrayBuffer /** diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/script/compaction/QLScriptCompaction.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/script/compaction/QLScriptCompaction.scala index 6e98d179bf..a9c0f7a340 100644 --- a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/script/compaction/QLScriptCompaction.scala +++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/script/compaction/QLScriptCompaction.scala @@ -25,6 +25,7 @@ class QLScriptCompaction private extends CommonScriptCompaction{ suffix match { case "sql" => true case "hql" => true + case "jdbc" => true case _ => false } } diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/source/AbstractFileSource.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/source/AbstractFileSource.scala index 0d08318dec..ee34e2485a 100644 --- a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/source/AbstractFileSource.scala +++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/source/AbstractFileSource.scala @@ -28,7 +28,7 @@ import org.apache.commons.math3.util.Pair import scala.collection.JavaConversions._ /** - * Created by patinousward on 2020/1/15. + * Created by johnnwang on 2020/1/15. */ abstract class AbstractFileSource extends FileSource { diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/source/FileSource.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/source/FileSource.scala index 9540b16515..05ecbea707 100644 --- a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/source/FileSource.scala +++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/source/FileSource.scala @@ -27,7 +27,7 @@ import com.webank.wedatasphere.linkis.storage.utils.StorageConfiguration import org.apache.commons.math3.util.Pair /** - * Created by patinousward on 2020/1/15. + * Created by johnnwang on 2020/1/15. */ trait FileSource extends Closeable { diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/source/ResultsetFileSource.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/source/ResultsetFileSource.scala index a82cb0b2ba..4e237e9eb9 100644 --- a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/source/ResultsetFileSource.scala +++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/source/ResultsetFileSource.scala @@ -20,7 +20,7 @@ import com.webank.wedatasphere.linkis.storage.resultset.table.TableRecord import com.webank.wedatasphere.linkis.storage.utils.StorageUtils /** - * Created by patinousward on 2020/1/15. + * Created by johnnwang on 2020/1/15. */ class ResultsetFileSource extends AbstractFileSource { diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/source/TextFileSource.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/source/TextFileSource.scala index 01f5fef1c9..0c3732930f 100644 --- a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/source/TextFileSource.scala +++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/source/TextFileSource.scala @@ -23,7 +23,7 @@ import com.webank.wedatasphere.linkis.storage.script.ScriptRecord import org.apache.commons.math3.util.Pair /** - * Created by patinousward on 2020/1/15. + * Created by johnnwang on 2020/1/15. */ class TextFileSource extends AbstractFileSource { diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/utils/StorageHelper.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/utils/StorageHelper.scala index 3878405ef5..c4b2f1555c 100644 --- a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/utils/StorageHelper.scala +++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/utils/StorageHelper.scala @@ -16,10 +16,9 @@ package com.webank.wedatasphere.linkis.storage.utils -import com.webank.wedatasphere.linkis.common.io.FsPath import com.webank.wedatasphere.linkis.storage.FSFactory -import com.webank.wedatasphere.linkis.storage.resultset.{ResultSetFactory, ResultSetReader} import com.webank.wedatasphere.linkis.storage.resultset.table.{TableMetaData, TableRecord} +import com.webank.wedatasphere.linkis.storage.resultset.{ResultSetFactory, ResultSetReader} /** * Created by johnnwang on 2019/3/7. diff --git a/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/conversions/FsConvertions.scala b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/conversions/FsConvertions.scala index 9858a60252..a936d7ed53 100644 --- a/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/conversions/FsConvertions.scala +++ b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/conversions/FsConvertions.scala @@ -30,8 +30,3 @@ object FsConvertions { } } -class FsClosable(fs: Fs) extends Closeable { - override def close(): Unit = { - fs.close() - } -} diff --git a/ujes/definedEngines/spark/entrance/bin/start-spark-entrance.sh b/ujes/definedEngines/spark/entrance/bin/start-spark-entrance.sh index 56cab6df00..80cc775a4a 100644 --- a/ujes/definedEngines/spark/entrance/bin/start-spark-entrance.sh +++ b/ujes/definedEngines/spark/entrance/bin/start-spark-entrance.sh @@ -26,7 +26,7 @@ if [[ -f "${SERVER_PID}" ]]; then fi fi -nohup java $SERVER_JAVA_OPTS -cp $HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out & +nohup java $SERVER_JAVA_OPTS -cp ../module/lib/*:$HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out & pid=$! if [[ -z "${pid}" ]]; then echo "server $SERVER_NAME start failed!" --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
