This is an automated email from the ASF dual-hosted git repository. aiceflower pushed a commit to branch release-0.9.4 in repository https://gitbox.apache.org/repos/asf/linkis.git
commit 63d838effbd3a786e47753bc38bf7f666e69ab83 Author: jaminlu <[email protected]> AuthorDate: Thu Jun 4 14:31:30 2020 +0800 #406 Put the code in module metadata manager together --- datasource/metadatamanager/server/bin/start-mdm.sh | 38 ++++ datasource/metadatamanager/server/bin/stop-mdm.sh | 47 +++++ .../{src/main/resources => conf}/application.yml | 0 .../{src/main/resources => conf}/linkis.properties | 0 .../{src/main/resources => conf}/log4j.properties | 0 .../server/{src/main/resources => conf}/log4j2.xml | 0 datasource/metadatamanager/server/pom.xml | 41 ++++ .../server/src/main/assembly/distribution.xml | 206 ++++++++++++++++++++ .../service/elasticsearch/bin/start-mdm-elastic.sh | 38 ++++ .../service/elasticsearch/bin/stop-mdm-elastic.sh | 47 +++++ .../conf}/application.yml | 4 +- .../conf}/linkis.properties | 3 - .../conf}/log4j.properties | 0 .../resources => elasticsearch/conf}/log4j2.xml | 0 .../{server => service/elasticsearch}/pom.xml | 66 +++++-- .../src/main/assembly/distribution.xml | 206 ++++++++++++++++++++ .../metadatamanager/service/ElasticConnection.java | 140 ++++++++++++++ .../service/ElasticMetaService.java | 87 +++++++++ .../service/ElasticParamsMapper.java | 32 ++++ .../service/receiver/ElasticReceiver.scala | 29 +++ .../service/hive/bin/start-mdm-hive.sh | 38 ++++ .../service/hive/bin/stop-mdm-hive.sh | 47 +++++ .../{src/main/resources => conf}/application.yml | 0 .../{src/main/resources => conf}/linkis.properties | 0 .../hive/conf}/log4j.properties | 0 .../resources => service/hive/conf}/log4j2.xml | 0 datasource/metadatamanager/service/hive/pom.xml | 41 ++++ .../hive/src/main/assembly/distribution.xml | 206 ++++++++++++++++++++ .../service/mysql/bin/start-mdm-mysql.sh | 38 ++++ .../service/mysql/bin/stop-mdm-mysql.sh | 47 +++++ .../main/resources => mysql/conf}/application.yml | 4 +- .../resources => mysql/conf}/linkis.properties | 3 - .../mysql/conf}/log4j.properties | 0 .../resources => service/mysql/conf}/log4j2.xml | 0 datasource/metadatamanager/service/mysql/pom.xml | 124 ++++++++++++ .../mysql/src/main/assembly/distribution.xml | 206 ++++++++++++++++++++ .../metadatamanager/service/SqlConnection.java | 210 +++++++++++++++++++++ .../metadatamanager/service/SqlMetaService.java | 79 ++++++++ .../metadatamanager/service/SqlParamsMapper.java | 38 ++++ .../service/receiver/SqlReceiver.scala | 28 +++ 40 files changed, 2064 insertions(+), 29 deletions(-) diff --git a/datasource/metadatamanager/server/bin/start-mdm.sh b/datasource/metadatamanager/server/bin/start-mdm.sh new file mode 100644 index 0000000000..80cc775a4a --- /dev/null +++ b/datasource/metadatamanager/server/bin/start-mdm.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +cd `dirname $0` +cd .. +HOME=`pwd` + +export SERVER_PID=$HOME/bin/linkis.pid +export SERVER_LOG_PATH=$HOME/logs +export SERVER_CLASS=com.webank.wedatasphere.linkis.DataWorkCloudApplication + +if test -z "$SERVER_HEAP_SIZE" +then + export SERVER_HEAP_SIZE="512M" +fi + +if test -z "$SERVER_JAVA_OPTS" +then + export SERVER_JAVA_OPTS=" -Xmx$SERVER_HEAP_SIZE -XX:+UseG1GC -Xloggc:$HOME/logs/linkis-gc.log" +fi + +if [[ -f "${SERVER_PID}" ]]; then + pid=$(cat ${SERVER_PID}) + if kill -0 ${pid} >/dev/null 2>&1; then + echo "Server is already running." + exit 1 + fi +fi + +nohup java $SERVER_JAVA_OPTS -cp ../module/lib/*:$HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out & +pid=$! +if [[ -z "${pid}" ]]; then + echo "server $SERVER_NAME start failed!" + exit 1 +else + echo "server $SERVER_NAME start succeeded!" + echo $pid > $SERVER_PID + sleep 1 +fi \ No newline at end of file diff --git a/datasource/metadatamanager/server/bin/stop-mdm.sh b/datasource/metadatamanager/server/bin/stop-mdm.sh new file mode 100644 index 0000000000..f032887111 --- /dev/null +++ b/datasource/metadatamanager/server/bin/stop-mdm.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +cd `dirname $0` +cd .. +HOME=`pwd` + +export SERVER_PID=$HOME/bin/linkis.pid + +function wait_for_server_to_die() { + local pid + local count + pid=$1 + timeout=$2 + count=0 + timeoutTime=$(date "+%s") + let "timeoutTime+=$timeout" + currentTime=$(date "+%s") + forceKill=1 + + while [[ $currentTime -lt $timeoutTime ]]; do + $(kill ${pid} > /dev/null 2> /dev/null) + if kill -0 ${pid} > /dev/null 2>&1; then + sleep 3 + else + forceKill=0 + break + fi + currentTime=$(date "+%s") + done + + if [[ forceKill -ne 0 ]]; then + $(kill -9 ${pid} > /dev/null 2> /dev/null) + fi +} + +if [[ ! -f "${SERVER_PID}" ]]; then + echo "server $SERVER_NAME is not running" +else + pid=$(cat ${SERVER_PID}) + if [[ -z "${pid}" ]]; then + echo "server $SERVER_NAME is not running" + else + wait_for_server_to_die $pid 40 + $(rm -f ${SERVER_PID}) + echo "server $SERVER_NAME is stopped." + fi +fi \ No newline at end of file diff --git a/datasource/metadatamanager/server/src/main/resources/application.yml b/datasource/metadatamanager/server/conf/application.yml similarity index 100% rename from datasource/metadatamanager/server/src/main/resources/application.yml rename to datasource/metadatamanager/server/conf/application.yml diff --git a/datasource/metadatamanager/server/src/main/resources/linkis.properties b/datasource/metadatamanager/server/conf/linkis.properties similarity index 100% rename from datasource/metadatamanager/server/src/main/resources/linkis.properties rename to datasource/metadatamanager/server/conf/linkis.properties diff --git a/datasource/metadatamanager/server/src/main/resources/log4j.properties b/datasource/metadatamanager/server/conf/log4j.properties similarity index 100% copy from datasource/metadatamanager/server/src/main/resources/log4j.properties copy to datasource/metadatamanager/server/conf/log4j.properties diff --git a/datasource/metadatamanager/server/src/main/resources/log4j2.xml b/datasource/metadatamanager/server/conf/log4j2.xml similarity index 100% copy from datasource/metadatamanager/server/src/main/resources/log4j2.xml copy to datasource/metadatamanager/server/conf/log4j2.xml diff --git a/datasource/metadatamanager/server/pom.xml b/datasource/metadatamanager/server/pom.xml index b9aa5f7fd2..51a2dc2c0b 100644 --- a/datasource/metadatamanager/server/pom.xml +++ b/datasource/metadatamanager/server/pom.xml @@ -80,10 +80,51 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-jar-plugin</artifactId> </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-assembly-plugin</artifactId> + <version>2.3</version> + <inherited>false</inherited> + <executions> + <execution> + <id>make-assembly</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + <configuration> + <descriptors> + <descriptor>src/main/assembly/distribution.xml</descriptor> + </descriptors> + </configuration> + </execution> + </executions> + <configuration> + <skipAssembly>false</skipAssembly> + <finalName>linkis-mdm-server</finalName> + <appendAssemblyId>false</appendAssemblyId> + <attach>false</attach> + <descriptors> + <descriptor>src/main/assembly/distribution.xml</descriptor> + </descriptors> + </configuration> + </plugin> </plugins> <resources> + <resource> + <directory>src/main/java</directory> + <includes> + <include>**/*.xml</include> + </includes> + </resource> <resource> <directory>src/main/resources</directory> + <excludes> + <exclude>**/*.properties</exclude> + <exclude>**/application.yml</exclude> + <exclude>**/bootstrap.yml</exclude> + <exclude>**/log4j2.xml</exclude> + </excludes> </resource> </resources> <finalName>${project.artifactId}-${project.version}</finalName> diff --git a/datasource/metadatamanager/server/src/main/assembly/distribution.xml b/datasource/metadatamanager/server/src/main/assembly/distribution.xml new file mode 100644 index 0000000000..56a7f09c3a --- /dev/null +++ b/datasource/metadatamanager/server/src/main/assembly/distribution.xml @@ -0,0 +1,206 @@ +<!-- + ~ Copyright 2019 WeBank + ~ + ~ Licensed under the Apache License, Version 2.0 (the "License"); + ~ you may not use this file except in compliance with the License. + ~ You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<assembly + xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/2.3"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/2.3 http://maven.apache.org/xsd/assembly-1.1.2.xsd";> + <id>linkis-mdm-server</id> + <formats> + <format>zip</format> + </formats> + <includeBaseDirectory>true</includeBaseDirectory> + <baseDirectory>linkis-mdm-server</baseDirectory> + + <dependencySets> + <dependencySet> + <!-- Enable access to all projects in the current multimodule build! <useAllReactorProjects>true</useAllReactorProjects> --> + <!-- Now, select which projects to include in this module-set. --> + <outputDirectory>lib</outputDirectory> + <useProjectArtifact>true</useProjectArtifact> + <useTransitiveDependencies>true</useTransitiveDependencies> + <unpack>false</unpack> + <useStrictFiltering>true</useStrictFiltering> + <useTransitiveFiltering>true</useTransitiveFiltering> + + <!-- <excludes> + <exclude>antlr:antlr:jar</exclude> + <exclude>aopalliance:aopalliance:jar</exclude> + <exclude>com.fasterxml.jackson.core:jackson-annotations:jar</exclude> + <exclude>com.fasterxml.jackson.core:jackson-core:jar</exclude> + <exclude>com.fasterxml.jackson.core:jackson-databind:jar</exclude> + <exclude>com.fasterxml.jackson.module:jackson-module-paranamer:jar</exclude> + <exclude>com.fasterxml.jackson.module:jackson-module-scala_2.11:jar</exclude> + <exclude>com.google.code.gson:gson:jar</exclude> + <exclude>com.google.guava:guava:jar</exclude> + <exclude>com.google.protobuf:protobuf-java:jar</exclude> + <exclude>com.netflix.archaius:archaius-core:jar</exclude> + <exclude>com.netflix.hystrix:hystrix-core:jar</exclude> + <exclude>com.netflix.netflix-commons:netflix-commons-util:jar</exclude> + <exclude>com.netflix.netflix-commons:netflix-statistics:jar</exclude> + <exclude>com.netflix.ribbon:ribbon:jar</exclude> + <exclude>com.netflix.ribbon:ribbon-core:jar</exclude> + <exclude>com.netflix.ribbon:ribbon-httpclient:jar</exclude> + <exclude>com.netflix.ribbon:ribbon-loadbalancer:jar</exclude> + <exclude>com.netflix.ribbon:ribbon-transport:jar</exclude> + <exclude>com.netflix.servo:servo-core:jar</exclude> + <exclude>com.sun.jersey.contribs:jersey-apache-client4:jar</exclude> + <exclude>com.sun.jersey:jersey-client:jar</exclude> + <exclude>com.sun.jersey:jersey-core:jar</exclude> + <exclude>com.sun.jersey:jersey-server:jar</exclude> + <exclude>com.sun.xml.bind:jaxb-impl:jar</exclude> + <exclude>com.webank.wedatasphere.linkis:linkis-common:jar</exclude> + <exclude>commons-beanutils:commons-beanutils:jar</exclude> + <exclude>commons-beanutils:commons-beanutils-core:jar</exclude> + <exclude>commons-cli:commons-cli:jar</exclude> + <exclude>commons-collections:commons-collections:jar</exclude> + <exclude>commons-configuration:commons-configuration:jar</exclude> + <exclude>commons-daemon:commons-daemon:jar</exclude> + <exclude>commons-dbcp:commons-dbcp:jar</exclude> + <exclude>commons-digester:commons-digester:jar</exclude> + <exclude>commons-io:commons-io:jar</exclude> + <exclude>commons-lang:commons-lang:jar</exclude> + <exclude>commons-net:commons-net:jar</exclude> + <exclude>commons-pool:commons-pool:jar</exclude> + <exclude>io.netty:netty:jar</exclude> + <exclude>io.netty:netty-all:jar</exclude> + <exclude>io.netty:netty-buffer:jar</exclude> + <exclude>io.netty:netty-codec:jar</exclude> + <exclude>io.netty:netty-codec-http:jar</exclude> + <exclude>io.netty:netty-common:jar</exclude> + <exclude>io.netty:netty-handler:jar</exclude> + <exclude>io.netty:netty-transport:jar</exclude> + <exclude>io.netty:netty-transport-native-epoll:jar</exclude> + <exclude>io.reactivex:rxjava:jar</exclude> + <exclude>io.reactivex:rxnetty:jar</exclude> + <exclude>io.reactivex:rxnetty-contexts:jar</exclude> + <exclude>io.reactivex:rxnetty-servo:jar</exclude> + <exclude>javax.activation:activation:jar</exclude> + <exclude>javax.annotation:javax.annotation-api:jar</exclude> + <exclude>javax.inject:javax.inject:jar</exclude> + <exclude>javax.servlet:javax.servlet-api:jar</exclude> + <exclude>javax.servlet.jsp:jsp-api:jar</exclude> + <exclude>javax.xml.bind:jaxb-api:jar</exclude> + <exclude>javax.xml.stream:stax-api:jar</exclude> + <exclude>net.databinder.dispatch:dispatch-core_2.11:jar</exclude> + <exclude>net.databinder.dispatch:dispatch-json4s-jackson_2.11:jar</exclude> + <exclude>org.antlr:antlr-runtime:jar</exclude> + <exclude>org.antlr:stringtemplate:jar</exclude> + <exclude>org.apache.commons:commons-compress:jar</exclude> + <exclude>org.apache.commons:commons-math3:jar</exclude> + <exclude>org.apache.curator:curator-framework:jar</exclude> + <exclude>org.apache.curator:curator-recipes:jar</exclude> + <exclude>org.apache.directory.api:api-asn1-api:jar</exclude> + <exclude>org.apache.directory.api:api-util:jar</exclude> + <exclude>org.apache.directory.server:apacheds-i18n:jar</exclude> + <exclude>org.apache.directory.server:apacheds-kerberos-codec:jar</exclude> + <exclude>org.apache.hadoop:hadoop-annotations:jar</exclude> + <exclude>org.apache.hadoop:hadoop-auth:jar</exclude> + <exclude>org.apache.hadoop:hadoop-common:jar</exclude> + <exclude>org.apache.hadoop:hadoop-hdfs:jar</exclude> + <exclude>org.apache.htrace:htrace-core:jar</exclude> + <exclude>org.apache.logging.log4j:log4j-api:jar</exclude> + <exclude>org.apache.zookeeper:zookeeper:jar</exclude> + <exclude>org.aspectj:aspectjweaver:jar</exclude> + <exclude>org.bouncycastle:bcpkix-jdk15on:jar</exclude> + <exclude>org.bouncycastle:bcprov-jdk15on:jar</exclude> + <exclude>org.codehaus.jettison:jettison:jar</exclude> + <exclude>org.eclipse.jetty:jetty-continuation:jar</exclude> + <exclude>org.eclipse.jetty:jetty-http:jar</exclude> + <exclude>org.eclipse.jetty:jetty-io:jar</exclude> + <exclude>org.eclipse.jetty:jetty-jndi:jar</exclude> + <exclude>org.eclipse.jetty:jetty-plus:jar</exclude> + <exclude>org.eclipse.jetty:jetty-security:jar</exclude> + <exclude>org.eclipse.jetty:jetty-server:jar</exclude> + <exclude>org.eclipse.jetty:jetty-servlet:jar</exclude> + <exclude>org.eclipse.jetty:jetty-util:jar</exclude> + <exclude>org.eclipse.jetty:jetty-webapp:jar</exclude> + <exclude>org.eclipse.jetty:jetty-xml:jar</exclude> + <exclude>org.fusesource.leveldbjni:leveldbjni-all:jar</exclude> + <exclude>org.hdrhistogram:HdrHistogram:jar</exclude> + <exclude>org.json4s:json4s-ast_2.11:jar</exclude> + <exclude>org.json4s:json4s-core_2.11:jar</exclude> + <exclude>org.json4s:json4s-jackson_2.11:jar</exclude> + <exclude>org.jsoup:jsoup:jar</exclude> + <exclude>org.mortbay.jetty:jetty:jar</exclude> + <exclude>org.mortbay.jetty:jetty-util:jar</exclude> + <exclude>org.scala-lang.modules:scala-parser-combinators_2.11:jar</exclude> + <exclude>org.scala-lang.modules:scala-xml_2.11:jar</exclude> + <exclude>org.scala-lang:scala-compiler:jar</exclude> + <exclude>org.scala-lang:scala-library:jar</exclude> + <exclude>org.scala-lang:scala-reflect:jar</exclude> + <exclude>org.scala-lang:scalap:jar</exclude> + <exclude>org.slf4j:jul-to-slf4j:jar</exclude> + <exclude>org.slf4j:slf4j-api:jar</exclude> + <exclude>org.springframework.boot:spring-boot:jar</exclude> + <exclude>org.springframework.boot:spring-boot-autoconfigure:jar</exclude> + <exclude>org.springframework.boot:spring-boot-starter:jar</exclude> + <exclude>org.springframework.boot:spring-boot-starter-aop:jar</exclude> + <exclude>org.springframework.cloud:spring-cloud-commons:jar</exclude> + <exclude>org.springframework.cloud:spring-cloud-context:jar</exclude> + <exclude>org.springframework.cloud:spring-cloud-netflix-archaius:jar</exclude> + <exclude>org.springframework.cloud:spring-cloud-netflix-ribbon:jar</exclude> + <exclude>org.springframework.cloud:spring-cloud-starter:jar</exclude> + <exclude>org.springframework.cloud:spring-cloud-starter-netflix-archaius:jar</exclude> + <exclude>org.springframework.cloud:spring-cloud-starter-netflix-ribbon:jar</exclude> + <exclude>org.springframework.security:spring-security-crypto:jar</exclude> + <exclude>org.springframework.security:spring-security-rsa:jar</exclude> + <exclude>org.springframework:spring-aop:jar</exclude> + <exclude>org.springframework:spring-beans:jar</exclude> + <exclude>org.springframework:spring-context:jar</exclude> + <exclude>org.springframework:spring-core:jar</exclude> + <exclude>org.springframework:spring-expression:jar</exclude> + <!–<exclude>org.springframework:spring-jcl:jar</exclude>–> + <exclude>org.springframework:spring-web:jar</exclude> + <exclude>org.tukaani:xz:jar</exclude> + <exclude>org.yaml:snakeyaml:jar</exclude> + <exclude>xerces:xercesImpl:jar</exclude> + <exclude>xml-apis:xml-apis:jar</exclude> + <exclude>xmlenc:xmlenc:jar</exclude> + </excludes>--> + </dependencySet> + </dependencySets> + + <fileSets> + <fileSet> + <directory>${basedir}/conf</directory> + <includes> + <include>*</include> + </includes> + <fileMode>0777</fileMode> + <outputDirectory>conf</outputDirectory> + <lineEnding>unix</lineEnding> + </fileSet> + <fileSet> + <directory>${basedir}/bin</directory> + <includes> + <include>*</include> + </includes> + <fileMode>0777</fileMode> + <outputDirectory>bin</outputDirectory> + <lineEnding>unix</lineEnding> + </fileSet> + <fileSet> + <directory>.</directory> + <excludes> + <exclude>*/**</exclude> + </excludes> + <outputDirectory>logs</outputDirectory> + </fileSet> + </fileSets> + +</assembly> + diff --git a/datasource/metadatamanager/service/elasticsearch/bin/start-mdm-elastic.sh b/datasource/metadatamanager/service/elasticsearch/bin/start-mdm-elastic.sh new file mode 100644 index 0000000000..80cc775a4a --- /dev/null +++ b/datasource/metadatamanager/service/elasticsearch/bin/start-mdm-elastic.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +cd `dirname $0` +cd .. +HOME=`pwd` + +export SERVER_PID=$HOME/bin/linkis.pid +export SERVER_LOG_PATH=$HOME/logs +export SERVER_CLASS=com.webank.wedatasphere.linkis.DataWorkCloudApplication + +if test -z "$SERVER_HEAP_SIZE" +then + export SERVER_HEAP_SIZE="512M" +fi + +if test -z "$SERVER_JAVA_OPTS" +then + export SERVER_JAVA_OPTS=" -Xmx$SERVER_HEAP_SIZE -XX:+UseG1GC -Xloggc:$HOME/logs/linkis-gc.log" +fi + +if [[ -f "${SERVER_PID}" ]]; then + pid=$(cat ${SERVER_PID}) + if kill -0 ${pid} >/dev/null 2>&1; then + echo "Server is already running." + exit 1 + fi +fi + +nohup java $SERVER_JAVA_OPTS -cp ../module/lib/*:$HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out & +pid=$! +if [[ -z "${pid}" ]]; then + echo "server $SERVER_NAME start failed!" + exit 1 +else + echo "server $SERVER_NAME start succeeded!" + echo $pid > $SERVER_PID + sleep 1 +fi \ No newline at end of file diff --git a/datasource/metadatamanager/service/elasticsearch/bin/stop-mdm-elastic.sh b/datasource/metadatamanager/service/elasticsearch/bin/stop-mdm-elastic.sh new file mode 100644 index 0000000000..f032887111 --- /dev/null +++ b/datasource/metadatamanager/service/elasticsearch/bin/stop-mdm-elastic.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +cd `dirname $0` +cd .. +HOME=`pwd` + +export SERVER_PID=$HOME/bin/linkis.pid + +function wait_for_server_to_die() { + local pid + local count + pid=$1 + timeout=$2 + count=0 + timeoutTime=$(date "+%s") + let "timeoutTime+=$timeout" + currentTime=$(date "+%s") + forceKill=1 + + while [[ $currentTime -lt $timeoutTime ]]; do + $(kill ${pid} > /dev/null 2> /dev/null) + if kill -0 ${pid} > /dev/null 2>&1; then + sleep 3 + else + forceKill=0 + break + fi + currentTime=$(date "+%s") + done + + if [[ forceKill -ne 0 ]]; then + $(kill -9 ${pid} > /dev/null 2> /dev/null) + fi +} + +if [[ ! -f "${SERVER_PID}" ]]; then + echo "server $SERVER_NAME is not running" +else + pid=$(cat ${SERVER_PID}) + if [[ -z "${pid}" ]]; then + echo "server $SERVER_NAME is not running" + else + wait_for_server_to_die $pid 40 + $(rm -f ${SERVER_PID}) + echo "server $SERVER_NAME is stopped." + fi +fi \ No newline at end of file diff --git a/datasource/metadatamanager/service/hive/src/main/resources/application.yml b/datasource/metadatamanager/service/elasticsearch/conf/application.yml similarity index 85% copy from datasource/metadatamanager/service/hive/src/main/resources/application.yml copy to datasource/metadatamanager/service/elasticsearch/conf/application.yml index 1b52e767cd..8b3f9ce44a 100644 --- a/datasource/metadatamanager/service/hive/src/main/resources/application.yml +++ b/datasource/metadatamanager/service/elasticsearch/conf/application.yml @@ -1,8 +1,8 @@ server: - port: 8293 + port: 8295 spring: application: - name: mdm-service-hive + name: mdm-service-elasticsearch eureka: client: diff --git a/datasource/metadatamanager/service/hive/src/main/resources/linkis.properties b/datasource/metadatamanager/service/elasticsearch/conf/linkis.properties similarity index 93% copy from datasource/metadatamanager/service/hive/src/main/resources/linkis.properties copy to datasource/metadatamanager/service/elasticsearch/conf/linkis.properties index edea49cf37..f83c10d281 100644 --- a/datasource/metadatamanager/service/hive/src/main/resources/linkis.properties +++ b/datasource/metadatamanager/service/elasticsearch/conf/linkis.properties @@ -19,6 +19,3 @@ wds.linkis.server.restful.scan.packages= #sit wds.linkis.server.version=v1 -#bml -wds.linkis.gateway.ip= -wds.linkis.gateway.port= \ No newline at end of file diff --git a/datasource/metadatamanager/service/hive/src/main/resources/log4j.properties b/datasource/metadatamanager/service/elasticsearch/conf/log4j.properties similarity index 100% rename from datasource/metadatamanager/service/hive/src/main/resources/log4j.properties rename to datasource/metadatamanager/service/elasticsearch/conf/log4j.properties diff --git a/datasource/metadatamanager/service/hive/src/main/resources/log4j2.xml b/datasource/metadatamanager/service/elasticsearch/conf/log4j2.xml similarity index 100% rename from datasource/metadatamanager/service/hive/src/main/resources/log4j2.xml rename to datasource/metadatamanager/service/elasticsearch/conf/log4j2.xml diff --git a/datasource/metadatamanager/server/pom.xml b/datasource/metadatamanager/service/elasticsearch/pom.xml similarity index 59% copy from datasource/metadatamanager/server/pom.xml copy to datasource/metadatamanager/service/elasticsearch/pom.xml index b9aa5f7fd2..9ac98d0233 100644 --- a/datasource/metadatamanager/server/pom.xml +++ b/datasource/metadatamanager/service/elasticsearch/pom.xml @@ -21,34 +21,22 @@ <version>0.9.4</version> </parent> <modelVersion>4.0.0</modelVersion> - <artifactId>linkis-metadatamanager-server</artifactId> + <artifactId>linkis-metadatamanager-service-es</artifactId> <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <es.version>6.7.1</es.version> </properties> <dependencies> - <dependency> - <groupId>com.webank.wedatasphere.linkis</groupId> - <artifactId>linkis-module</artifactId> - <version>${linkis.version}</version> - <exclusions> - <exclusion> - <artifactId>asm</artifactId> - <groupId>org.ow2.asm</groupId> - </exclusion> - </exclusions> - </dependency> - <!--Metadata common--> <dependency> <groupId>com.webank.wedatasphere.linkis</groupId> <artifactId>linkis-metadatamanager-common</artifactId> <version>${linkis.version}</version> </dependency> - <!-- data source manager common dependency--> <dependency> <groupId>com.webank.wedatasphere.linkis</groupId> - <artifactId>linkis-datasourcemanager-common</artifactId> + <artifactId>linkis-module</artifactId> <version>${linkis.version}</version> <exclusions> <exclusion> @@ -57,11 +45,10 @@ </exclusion> </exclusions> </dependency> - <!--rpc--> <dependency> - <groupId>com.webank.wedatasphere.linkis</groupId> - <artifactId>linkis-cloudRPC</artifactId> - <version>${linkis.version}</version> + <groupId>org.elasticsearch.client</groupId> + <artifactId>elasticsearch-rest-client</artifactId> + <version>${es.version}</version> </dependency> </dependencies> @@ -80,10 +67,51 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-jar-plugin</artifactId> </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-assembly-plugin</artifactId> + <version>2.3</version> + <inherited>false</inherited> + <executions> + <execution> + <id>make-assembly</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + <configuration> + <descriptors> + <descriptor>src/main/assembly/distribution.xml</descriptor> + </descriptors> + </configuration> + </execution> + </executions> + <configuration> + <skipAssembly>false</skipAssembly> + <finalName>linkis-mdm-service-es</finalName> + <appendAssemblyId>false</appendAssemblyId> + <attach>false</attach> + <descriptors> + <descriptor>src/main/assembly/distribution.xml</descriptor> + </descriptors> + </configuration> + </plugin> </plugins> <resources> + <resource> + <directory>src/main/java</directory> + <includes> + <include>**/*.xml</include> + </includes> + </resource> <resource> <directory>src/main/resources</directory> + <excludes> + <exclude>**/*.properties</exclude> + <exclude>**/application.yml</exclude> + <exclude>**/bootstrap.yml</exclude> + <exclude>**/log4j2.xml</exclude> + </excludes> </resource> </resources> <finalName>${project.artifactId}-${project.version}</finalName> diff --git a/datasource/metadatamanager/service/elasticsearch/src/main/assembly/distribution.xml b/datasource/metadatamanager/service/elasticsearch/src/main/assembly/distribution.xml new file mode 100644 index 0000000000..f95f84e9ac --- /dev/null +++ b/datasource/metadatamanager/service/elasticsearch/src/main/assembly/distribution.xml @@ -0,0 +1,206 @@ +<!-- + ~ Copyright 2019 WeBank + ~ + ~ Licensed under the Apache License, Version 2.0 (the "License"); + ~ you may not use this file except in compliance with the License. + ~ You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<assembly + xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/2.3"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/2.3 http://maven.apache.org/xsd/assembly-1.1.2.xsd";> + <id>linkis-mdm-service-es</id> + <formats> + <format>zip</format> + </formats> + <includeBaseDirectory>true</includeBaseDirectory> + <baseDirectory>linkis-mdm-service-es</baseDirectory> + + <dependencySets> + <dependencySet> + <!-- Enable access to all projects in the current multimodule build! <useAllReactorProjects>true</useAllReactorProjects> --> + <!-- Now, select which projects to include in this module-set. --> + <outputDirectory>lib</outputDirectory> + <useProjectArtifact>true</useProjectArtifact> + <useTransitiveDependencies>true</useTransitiveDependencies> + <unpack>false</unpack> + <useStrictFiltering>true</useStrictFiltering> + <useTransitiveFiltering>true</useTransitiveFiltering> + + <!-- <excludes> + <exclude>antlr:antlr:jar</exclude> + <exclude>aopalliance:aopalliance:jar</exclude> + <exclude>com.fasterxml.jackson.core:jackson-annotations:jar</exclude> + <exclude>com.fasterxml.jackson.core:jackson-core:jar</exclude> + <exclude>com.fasterxml.jackson.core:jackson-databind:jar</exclude> + <exclude>com.fasterxml.jackson.module:jackson-module-paranamer:jar</exclude> + <exclude>com.fasterxml.jackson.module:jackson-module-scala_2.11:jar</exclude> + <exclude>com.google.code.gson:gson:jar</exclude> + <exclude>com.google.guava:guava:jar</exclude> + <exclude>com.google.protobuf:protobuf-java:jar</exclude> + <exclude>com.netflix.archaius:archaius-core:jar</exclude> + <exclude>com.netflix.hystrix:hystrix-core:jar</exclude> + <exclude>com.netflix.netflix-commons:netflix-commons-util:jar</exclude> + <exclude>com.netflix.netflix-commons:netflix-statistics:jar</exclude> + <exclude>com.netflix.ribbon:ribbon:jar</exclude> + <exclude>com.netflix.ribbon:ribbon-core:jar</exclude> + <exclude>com.netflix.ribbon:ribbon-httpclient:jar</exclude> + <exclude>com.netflix.ribbon:ribbon-loadbalancer:jar</exclude> + <exclude>com.netflix.ribbon:ribbon-transport:jar</exclude> + <exclude>com.netflix.servo:servo-core:jar</exclude> + <exclude>com.sun.jersey.contribs:jersey-apache-client4:jar</exclude> + <exclude>com.sun.jersey:jersey-client:jar</exclude> + <exclude>com.sun.jersey:jersey-core:jar</exclude> + <exclude>com.sun.jersey:jersey-server:jar</exclude> + <exclude>com.sun.xml.bind:jaxb-impl:jar</exclude> + <exclude>com.webank.wedatasphere.linkis:linkis-common:jar</exclude> + <exclude>commons-beanutils:commons-beanutils:jar</exclude> + <exclude>commons-beanutils:commons-beanutils-core:jar</exclude> + <exclude>commons-cli:commons-cli:jar</exclude> + <exclude>commons-collections:commons-collections:jar</exclude> + <exclude>commons-configuration:commons-configuration:jar</exclude> + <exclude>commons-daemon:commons-daemon:jar</exclude> + <exclude>commons-dbcp:commons-dbcp:jar</exclude> + <exclude>commons-digester:commons-digester:jar</exclude> + <exclude>commons-io:commons-io:jar</exclude> + <exclude>commons-lang:commons-lang:jar</exclude> + <exclude>commons-net:commons-net:jar</exclude> + <exclude>commons-pool:commons-pool:jar</exclude> + <exclude>io.netty:netty:jar</exclude> + <exclude>io.netty:netty-all:jar</exclude> + <exclude>io.netty:netty-buffer:jar</exclude> + <exclude>io.netty:netty-codec:jar</exclude> + <exclude>io.netty:netty-codec-http:jar</exclude> + <exclude>io.netty:netty-common:jar</exclude> + <exclude>io.netty:netty-handler:jar</exclude> + <exclude>io.netty:netty-transport:jar</exclude> + <exclude>io.netty:netty-transport-native-epoll:jar</exclude> + <exclude>io.reactivex:rxjava:jar</exclude> + <exclude>io.reactivex:rxnetty:jar</exclude> + <exclude>io.reactivex:rxnetty-contexts:jar</exclude> + <exclude>io.reactivex:rxnetty-servo:jar</exclude> + <exclude>javax.activation:activation:jar</exclude> + <exclude>javax.annotation:javax.annotation-api:jar</exclude> + <exclude>javax.inject:javax.inject:jar</exclude> + <exclude>javax.servlet:javax.servlet-api:jar</exclude> + <exclude>javax.servlet.jsp:jsp-api:jar</exclude> + <exclude>javax.xml.bind:jaxb-api:jar</exclude> + <exclude>javax.xml.stream:stax-api:jar</exclude> + <exclude>net.databinder.dispatch:dispatch-core_2.11:jar</exclude> + <exclude>net.databinder.dispatch:dispatch-json4s-jackson_2.11:jar</exclude> + <exclude>org.antlr:antlr-runtime:jar</exclude> + <exclude>org.antlr:stringtemplate:jar</exclude> + <exclude>org.apache.commons:commons-compress:jar</exclude> + <exclude>org.apache.commons:commons-math3:jar</exclude> + <exclude>org.apache.curator:curator-framework:jar</exclude> + <exclude>org.apache.curator:curator-recipes:jar</exclude> + <exclude>org.apache.directory.api:api-asn1-api:jar</exclude> + <exclude>org.apache.directory.api:api-util:jar</exclude> + <exclude>org.apache.directory.server:apacheds-i18n:jar</exclude> + <exclude>org.apache.directory.server:apacheds-kerberos-codec:jar</exclude> + <exclude>org.apache.hadoop:hadoop-annotations:jar</exclude> + <exclude>org.apache.hadoop:hadoop-auth:jar</exclude> + <exclude>org.apache.hadoop:hadoop-common:jar</exclude> + <exclude>org.apache.hadoop:hadoop-hdfs:jar</exclude> + <exclude>org.apache.htrace:htrace-core:jar</exclude> + <exclude>org.apache.logging.log4j:log4j-api:jar</exclude> + <exclude>org.apache.zookeeper:zookeeper:jar</exclude> + <exclude>org.aspectj:aspectjweaver:jar</exclude> + <exclude>org.bouncycastle:bcpkix-jdk15on:jar</exclude> + <exclude>org.bouncycastle:bcprov-jdk15on:jar</exclude> + <exclude>org.codehaus.jettison:jettison:jar</exclude> + <exclude>org.eclipse.jetty:jetty-continuation:jar</exclude> + <exclude>org.eclipse.jetty:jetty-http:jar</exclude> + <exclude>org.eclipse.jetty:jetty-io:jar</exclude> + <exclude>org.eclipse.jetty:jetty-jndi:jar</exclude> + <exclude>org.eclipse.jetty:jetty-plus:jar</exclude> + <exclude>org.eclipse.jetty:jetty-security:jar</exclude> + <exclude>org.eclipse.jetty:jetty-server:jar</exclude> + <exclude>org.eclipse.jetty:jetty-servlet:jar</exclude> + <exclude>org.eclipse.jetty:jetty-util:jar</exclude> + <exclude>org.eclipse.jetty:jetty-webapp:jar</exclude> + <exclude>org.eclipse.jetty:jetty-xml:jar</exclude> + <exclude>org.fusesource.leveldbjni:leveldbjni-all:jar</exclude> + <exclude>org.hdrhistogram:HdrHistogram:jar</exclude> + <exclude>org.json4s:json4s-ast_2.11:jar</exclude> + <exclude>org.json4s:json4s-core_2.11:jar</exclude> + <exclude>org.json4s:json4s-jackson_2.11:jar</exclude> + <exclude>org.jsoup:jsoup:jar</exclude> + <exclude>org.mortbay.jetty:jetty:jar</exclude> + <exclude>org.mortbay.jetty:jetty-util:jar</exclude> + <exclude>org.scala-lang.modules:scala-parser-combinators_2.11:jar</exclude> + <exclude>org.scala-lang.modules:scala-xml_2.11:jar</exclude> + <exclude>org.scala-lang:scala-compiler:jar</exclude> + <exclude>org.scala-lang:scala-library:jar</exclude> + <exclude>org.scala-lang:scala-reflect:jar</exclude> + <exclude>org.scala-lang:scalap:jar</exclude> + <exclude>org.slf4j:jul-to-slf4j:jar</exclude> + <exclude>org.slf4j:slf4j-api:jar</exclude> + <exclude>org.springframework.boot:spring-boot:jar</exclude> + <exclude>org.springframework.boot:spring-boot-autoconfigure:jar</exclude> + <exclude>org.springframework.boot:spring-boot-starter:jar</exclude> + <exclude>org.springframework.boot:spring-boot-starter-aop:jar</exclude> + <exclude>org.springframework.cloud:spring-cloud-commons:jar</exclude> + <exclude>org.springframework.cloud:spring-cloud-context:jar</exclude> + <exclude>org.springframework.cloud:spring-cloud-netflix-archaius:jar</exclude> + <exclude>org.springframework.cloud:spring-cloud-netflix-ribbon:jar</exclude> + <exclude>org.springframework.cloud:spring-cloud-starter:jar</exclude> + <exclude>org.springframework.cloud:spring-cloud-starter-netflix-archaius:jar</exclude> + <exclude>org.springframework.cloud:spring-cloud-starter-netflix-ribbon:jar</exclude> + <exclude>org.springframework.security:spring-security-crypto:jar</exclude> + <exclude>org.springframework.security:spring-security-rsa:jar</exclude> + <exclude>org.springframework:spring-aop:jar</exclude> + <exclude>org.springframework:spring-beans:jar</exclude> + <exclude>org.springframework:spring-context:jar</exclude> + <exclude>org.springframework:spring-core:jar</exclude> + <exclude>org.springframework:spring-expression:jar</exclude> + <!–<exclude>org.springframework:spring-jcl:jar</exclude>–> + <exclude>org.springframework:spring-web:jar</exclude> + <exclude>org.tukaani:xz:jar</exclude> + <exclude>org.yaml:snakeyaml:jar</exclude> + <exclude>xerces:xercesImpl:jar</exclude> + <exclude>xml-apis:xml-apis:jar</exclude> + <exclude>xmlenc:xmlenc:jar</exclude> + </excludes>--> + </dependencySet> + </dependencySets> + + <fileSets> + <fileSet> + <directory>${basedir}/conf</directory> + <includes> + <include>*</include> + </includes> + <fileMode>0777</fileMode> + <outputDirectory>conf</outputDirectory> + <lineEnding>unix</lineEnding> + </fileSet> + <fileSet> + <directory>${basedir}/bin</directory> + <includes> + <include>*</include> + </includes> + <fileMode>0777</fileMode> + <outputDirectory>bin</outputDirectory> + <lineEnding>unix</lineEnding> + </fileSet> + <fileSet> + <directory>.</directory> + <excludes> + <exclude>*/**</exclude> + </excludes> + <outputDirectory>logs</outputDirectory> + </fileSet> + </fileSets> + +</assembly> + diff --git a/datasource/metadatamanager/service/elasticsearch/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/ElasticConnection.java b/datasource/metadatamanager/service/elasticsearch/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/ElasticConnection.java new file mode 100644 index 0000000000..b485c69a3a --- /dev/null +++ b/datasource/metadatamanager/service/elasticsearch/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/ElasticConnection.java @@ -0,0 +1,140 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.metadatamanager.service; + +import com.webank.wedatasphere.linkis.metadatamanager.common.Json; +import org.apache.commons.lang.StringUtils; +import org.apache.http.HttpHost; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.nio.reactor.IOReactorConfig; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; + +import java.io.Closeable; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * @author davidhua + * 2020/02/14 + */ +public class ElasticConnection implements Closeable { + + public static final String DEFAULT_TYPE_NAME = "type"; + + private static final String DEFAULT_MAPPING_NAME = "mappings"; + private static final String DEFAULT_INDEX_NAME = "index"; + private static final String FIELD_PROPS = "properties"; + + private RestClient restClient; + + public ElasticConnection(String[] endPoints, String username, String password) throws IOException { + HttpHost[] httpHosts = new HttpHost[endPoints.length]; + for(int i = 0; i < endPoints.length; i++){ + httpHosts[i] = HttpHost.create(endPoints[i]); + } + RestClientBuilder restClientBuilder = RestClient.builder(httpHosts); + CredentialsProvider credentialsProvider = null; + if(StringUtils.isNotBlank(username) && StringUtils.isNotBlank(password)){ + credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, + new UsernamePasswordCredentials(username, password)); + } + //set only one thread + CredentialsProvider finalCredentialsProvider = credentialsProvider; + restClientBuilder.setHttpClientConfigCallback(httpClientBuilder -> { + if(null != finalCredentialsProvider){ + httpClientBuilder + .setDefaultCredentialsProvider(finalCredentialsProvider); + } + return httpClientBuilder.setDefaultIOReactorConfig(IOReactorConfig.custom().setIoThreadCount(1).build()); + }); + this.restClient = restClientBuilder.build(); + //Try to test connection + ping(); + } + + public List<String> getAllIndices() throws IOException { + List<String> indices = new ArrayList<>(); + Request request = new Request("GET", "_cat/indices"); + request.addParameter("format", "JSON"); + Response response = restClient.performRequest(request); + List<Map<String, Object>> list = Json.fromJson(response.getEntity().getContent(), Map.class); + list.forEach( v ->{ + String index = String.valueOf(v.getOrDefault(DEFAULT_INDEX_NAME, "")); + if(StringUtils.isNotBlank(index) && !index.startsWith(".")){ + indices.add(index); + } + }); + return indices; + } + + public List<String> getTypes(String index) throws IOException{ + List<String> types = new ArrayList<>(); + Request request = new Request("GET", index +"/_mappings"); + Response response = restClient.performRequest(request); + Map<String, Map<String, Object>> result = + Json.fromJson(response.getEntity().getContent(), Map.class); + Map<String, Object> indexMap = result.get(index); + Object props = indexMap.get(DEFAULT_MAPPING_NAME); + if(props instanceof Map){ + Set keySet = ((Map)props).keySet(); + for(Object v : keySet){ + types.add(String.valueOf(v)); + } + } + return types; + } + + public Map<Object, Object> getProps(String index, String type) throws IOException{ + Request request = new Request("GET", index + "/_mappings/" + type); + Response response = restClient.performRequest(request); + Map<String, Map<String, Object>> result = + Json.fromJson(response.getEntity().getContent(), Map.class); + Map mappings = (Map)result.get(index).get("mappings"); + Map propsMap = mappings; + if(mappings.containsKey(type)){ + Object typeMap = mappings.get(type); + if(typeMap instanceof Map){ + propsMap = (Map)typeMap; + } + } + Object props = propsMap.get(FIELD_PROPS); + if(props instanceof Map){ + return (Map)props; + } + return null; + } + public void ping() throws IOException{ + Response response = restClient.performRequest(new Request("GET", "/")); + int code = response.getStatusLine().getStatusCode(); + int successCode = 200; + if(code != successCode){ + throw new RuntimeException("Ping to ElasticSearch ERROR, response code: " + code); + } + } + + @Override + public void close() throws IOException { + this.restClient.close(); + } +} diff --git a/datasource/metadatamanager/service/elasticsearch/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/ElasticMetaService.java b/datasource/metadatamanager/service/elasticsearch/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/ElasticMetaService.java new file mode 100644 index 0000000000..8778ba42eb --- /dev/null +++ b/datasource/metadatamanager/service/elasticsearch/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/ElasticMetaService.java @@ -0,0 +1,87 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.metadatamanager.service; + +import com.webank.wedatasphere.linkis.metadatamanager.common.Json; +import com.webank.wedatasphere.linkis.metadatamanager.common.domain.MetaColumnInfo; +import com.webank.wedatasphere.linkis.metadatamanager.common.service.AbstractMetaService; +import com.webank.wedatasphere.linkis.metadatamanager.common.service.MetadataConnection; +import org.springframework.stereotype.Component; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * @author davidhua + * 2020/02/14 + */ +@Component +public class ElasticMetaService extends AbstractMetaService<ElasticConnection> { + @Override + public MetadataConnection<ElasticConnection> getConnection(String operator, Map<String, Object> params) + throws Exception{ + String[] endPoints = new String[]{}; + Object urls = params.get(ElasticParamsMapper.PARAM_ES_URLS.getValue()); + if(!(urls instanceof List)){ + List<String> urlList = Json.fromJson(String.valueOf(urls), List.class, String.class); + assert urlList != null; + endPoints = urlList.toArray(endPoints); + }else{ + endPoints = ((List<String>)urls).toArray(endPoints); + } + ElasticConnection conn = new ElasticConnection(endPoints, String.valueOf(params.getOrDefault(ElasticParamsMapper.PARAM_ES_USERNAME.getValue(), "")), + String.valueOf(params.getOrDefault(ElasticParamsMapper.PARAM_ES_PASSWORD.getValue(), ""))); + return new MetadataConnection<>(conn, false); + } + + @Override + public List<String> queryDatabases(ElasticConnection connection) { + //Get indices + try{ + return connection.getAllIndices(); + }catch (Exception e){ + throw new RuntimeException("Fail to get ElasticSearch indices(获取索引列表失败)", e); + } + } + + @Override + public List<String> queryTables(ElasticConnection connection, String database) { + //Get types + try{ + return connection.getTypes(database); + }catch (Exception e){ + throw new RuntimeException("Fail to get ElasticSearch types(获取索引类型失败)", e); + } + } + + @Override + public List<MetaColumnInfo> queryColumns(ElasticConnection connection, String database, String table) { + try { + Map<Object, Object> props = connection.getProps(database, table); + return props.entrySet().stream().map(entry -> { + MetaColumnInfo info = new MetaColumnInfo(); + info.setName(String.valueOf(entry.getKey())); + Object value = entry.getValue(); + if(value instanceof Map){ + info.setType(String.valueOf(((Map)value) + .getOrDefault(ElasticConnection.DEFAULT_TYPE_NAME, ""))); + } + return info; + }).collect(Collectors.toList()); + } catch (Exception e) { + throw new RuntimeException("Fail to get ElasticSearch columns(获取索引字段失败)", e); + } + } +} diff --git a/datasource/metadatamanager/service/elasticsearch/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/ElasticParamsMapper.java b/datasource/metadatamanager/service/elasticsearch/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/ElasticParamsMapper.java new file mode 100644 index 0000000000..3af7e8ad86 --- /dev/null +++ b/datasource/metadatamanager/service/elasticsearch/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/ElasticParamsMapper.java @@ -0,0 +1,32 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.metadatamanager.service; + +import com.webank.wedatasphere.linkis.common.conf.CommonVars; + +/** + * Configuration + * @author davidhua + * 2020/02/14 + */ +public class ElasticParamsMapper { + public static final CommonVars<String> PARAM_ES_URLS = + CommonVars.apply("wds.linkis.server.mdm.service.es.urls", "elasticUrls"); + + public static final CommonVars<String> PARAM_ES_USERNAME = + CommonVars.apply("wds.linkis.server.mdm.service.es.username", "username"); + + public static final CommonVars<String> PARAM_ES_PASSWORD = + CommonVars.apply("wds.linkis.server.mdm.service.es.password", "password"); +} diff --git a/datasource/metadatamanager/service/elasticsearch/src/main/scala/com/webank/wedatasphere/linkis/metadatamanager/service/receiver/ElasticReceiver.scala b/datasource/metadatamanager/service/elasticsearch/src/main/scala/com/webank/wedatasphere/linkis/metadatamanager/service/receiver/ElasticReceiver.scala new file mode 100644 index 0000000000..dd3049f1de --- /dev/null +++ b/datasource/metadatamanager/service/elasticsearch/src/main/scala/com/webank/wedatasphere/linkis/metadatamanager/service/receiver/ElasticReceiver.scala @@ -0,0 +1,29 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.metadatamanager.service.receiver + +import com.webank.wedatasphere.linkis.DataWorkCloudApplication +import com.webank.wedatasphere.linkis.metadatamanager.common.service.MetadataService +import com.webank.wedatasphere.linkis.metadatamanager.common.receiver.BaseMetaReceiver +import com.webank.wedatasphere.linkis.metadatamanager.common.service.MetadataService +import javax.annotation.PostConstruct +import org.springframework.stereotype.Component + +@Component +class ElasticReceiver extends BaseMetaReceiver{ + @PostConstruct + def init(): Unit = { + metadataService = DataWorkCloudApplication.getApplicationContext.getBean(classOf[MetadataService]) + } +} diff --git a/datasource/metadatamanager/service/hive/bin/start-mdm-hive.sh b/datasource/metadatamanager/service/hive/bin/start-mdm-hive.sh new file mode 100644 index 0000000000..80cc775a4a --- /dev/null +++ b/datasource/metadatamanager/service/hive/bin/start-mdm-hive.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +cd `dirname $0` +cd .. +HOME=`pwd` + +export SERVER_PID=$HOME/bin/linkis.pid +export SERVER_LOG_PATH=$HOME/logs +export SERVER_CLASS=com.webank.wedatasphere.linkis.DataWorkCloudApplication + +if test -z "$SERVER_HEAP_SIZE" +then + export SERVER_HEAP_SIZE="512M" +fi + +if test -z "$SERVER_JAVA_OPTS" +then + export SERVER_JAVA_OPTS=" -Xmx$SERVER_HEAP_SIZE -XX:+UseG1GC -Xloggc:$HOME/logs/linkis-gc.log" +fi + +if [[ -f "${SERVER_PID}" ]]; then + pid=$(cat ${SERVER_PID}) + if kill -0 ${pid} >/dev/null 2>&1; then + echo "Server is already running." + exit 1 + fi +fi + +nohup java $SERVER_JAVA_OPTS -cp ../module/lib/*:$HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out & +pid=$! +if [[ -z "${pid}" ]]; then + echo "server $SERVER_NAME start failed!" + exit 1 +else + echo "server $SERVER_NAME start succeeded!" + echo $pid > $SERVER_PID + sleep 1 +fi \ No newline at end of file diff --git a/datasource/metadatamanager/service/hive/bin/stop-mdm-hive.sh b/datasource/metadatamanager/service/hive/bin/stop-mdm-hive.sh new file mode 100644 index 0000000000..f032887111 --- /dev/null +++ b/datasource/metadatamanager/service/hive/bin/stop-mdm-hive.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +cd `dirname $0` +cd .. +HOME=`pwd` + +export SERVER_PID=$HOME/bin/linkis.pid + +function wait_for_server_to_die() { + local pid + local count + pid=$1 + timeout=$2 + count=0 + timeoutTime=$(date "+%s") + let "timeoutTime+=$timeout" + currentTime=$(date "+%s") + forceKill=1 + + while [[ $currentTime -lt $timeoutTime ]]; do + $(kill ${pid} > /dev/null 2> /dev/null) + if kill -0 ${pid} > /dev/null 2>&1; then + sleep 3 + else + forceKill=0 + break + fi + currentTime=$(date "+%s") + done + + if [[ forceKill -ne 0 ]]; then + $(kill -9 ${pid} > /dev/null 2> /dev/null) + fi +} + +if [[ ! -f "${SERVER_PID}" ]]; then + echo "server $SERVER_NAME is not running" +else + pid=$(cat ${SERVER_PID}) + if [[ -z "${pid}" ]]; then + echo "server $SERVER_NAME is not running" + else + wait_for_server_to_die $pid 40 + $(rm -f ${SERVER_PID}) + echo "server $SERVER_NAME is stopped." + fi +fi \ No newline at end of file diff --git a/datasource/metadatamanager/service/hive/src/main/resources/application.yml b/datasource/metadatamanager/service/hive/conf/application.yml similarity index 100% copy from datasource/metadatamanager/service/hive/src/main/resources/application.yml copy to datasource/metadatamanager/service/hive/conf/application.yml diff --git a/datasource/metadatamanager/service/hive/src/main/resources/linkis.properties b/datasource/metadatamanager/service/hive/conf/linkis.properties similarity index 100% copy from datasource/metadatamanager/service/hive/src/main/resources/linkis.properties copy to datasource/metadatamanager/service/hive/conf/linkis.properties diff --git a/datasource/metadatamanager/server/src/main/resources/log4j.properties b/datasource/metadatamanager/service/hive/conf/log4j.properties similarity index 100% copy from datasource/metadatamanager/server/src/main/resources/log4j.properties copy to datasource/metadatamanager/service/hive/conf/log4j.properties diff --git a/datasource/metadatamanager/server/src/main/resources/log4j2.xml b/datasource/metadatamanager/service/hive/conf/log4j2.xml similarity index 100% copy from datasource/metadatamanager/server/src/main/resources/log4j2.xml copy to datasource/metadatamanager/service/hive/conf/log4j2.xml diff --git a/datasource/metadatamanager/service/hive/pom.xml b/datasource/metadatamanager/service/hive/pom.xml index 1f92b4514e..20a3880e83 100644 --- a/datasource/metadatamanager/service/hive/pom.xml +++ b/datasource/metadatamanager/service/hive/pom.xml @@ -106,10 +106,51 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-jar-plugin</artifactId> </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-assembly-plugin</artifactId> + <version>2.3</version> + <inherited>false</inherited> + <executions> + <execution> + <id>make-assembly</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + <configuration> + <descriptors> + <descriptor>src/main/assembly/distribution.xml</descriptor> + </descriptors> + </configuration> + </execution> + </executions> + <configuration> + <skipAssembly>false</skipAssembly> + <finalName>linkis-mdm-service-hive</finalName> + <appendAssemblyId>false</appendAssemblyId> + <attach>false</attach> + <descriptors> + <descriptor>src/main/assembly/distribution.xml</descriptor> + </descriptors> + </configuration> + </plugin> </plugins> <resources> + <resource> + <directory>src/main/java</directory> + <includes> + <include>**/*.xml</include> + </includes> + </resource> <resource> <directory>src/main/resources</directory> + <excludes> + <exclude>**/*.properties</exclude> + <exclude>**/application.yml</exclude> + <exclude>**/bootstrap.yml</exclude> + <exclude>**/log4j2.xml</exclude> + </excludes> </resource> </resources> <finalName>${project.artifactId}-${project.version}</finalName> diff --git a/datasource/metadatamanager/service/hive/src/main/assembly/distribution.xml b/datasource/metadatamanager/service/hive/src/main/assembly/distribution.xml new file mode 100644 index 0000000000..234308bd1c --- /dev/null +++ b/datasource/metadatamanager/service/hive/src/main/assembly/distribution.xml @@ -0,0 +1,206 @@ +<!-- + ~ Copyright 2019 WeBank + ~ + ~ Licensed under the Apache License, Version 2.0 (the "License"); + ~ you may not use this file except in compliance with the License. + ~ You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<assembly + xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/2.3"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/2.3 http://maven.apache.org/xsd/assembly-1.1.2.xsd";> + <id>linkis-mdm-service-hive</id> + <formats> + <format>zip</format> + </formats> + <includeBaseDirectory>true</includeBaseDirectory> + <baseDirectory>linkis-mdm-service-hive</baseDirectory> + + <dependencySets> + <dependencySet> + <!-- Enable access to all projects in the current multimodule build! <useAllReactorProjects>true</useAllReactorProjects> --> + <!-- Now, select which projects to include in this module-set. --> + <outputDirectory>lib</outputDirectory> + <useProjectArtifact>true</useProjectArtifact> + <useTransitiveDependencies>true</useTransitiveDependencies> + <unpack>false</unpack> + <useStrictFiltering>true</useStrictFiltering> + <useTransitiveFiltering>true</useTransitiveFiltering> + + <!-- <excludes> + <exclude>antlr:antlr:jar</exclude> + <exclude>aopalliance:aopalliance:jar</exclude> + <exclude>com.fasterxml.jackson.core:jackson-annotations:jar</exclude> + <exclude>com.fasterxml.jackson.core:jackson-core:jar</exclude> + <exclude>com.fasterxml.jackson.core:jackson-databind:jar</exclude> + <exclude>com.fasterxml.jackson.module:jackson-module-paranamer:jar</exclude> + <exclude>com.fasterxml.jackson.module:jackson-module-scala_2.11:jar</exclude> + <exclude>com.google.code.gson:gson:jar</exclude> + <exclude>com.google.guava:guava:jar</exclude> + <exclude>com.google.protobuf:protobuf-java:jar</exclude> + <exclude>com.netflix.archaius:archaius-core:jar</exclude> + <exclude>com.netflix.hystrix:hystrix-core:jar</exclude> + <exclude>com.netflix.netflix-commons:netflix-commons-util:jar</exclude> + <exclude>com.netflix.netflix-commons:netflix-statistics:jar</exclude> + <exclude>com.netflix.ribbon:ribbon:jar</exclude> + <exclude>com.netflix.ribbon:ribbon-core:jar</exclude> + <exclude>com.netflix.ribbon:ribbon-httpclient:jar</exclude> + <exclude>com.netflix.ribbon:ribbon-loadbalancer:jar</exclude> + <exclude>com.netflix.ribbon:ribbon-transport:jar</exclude> + <exclude>com.netflix.servo:servo-core:jar</exclude> + <exclude>com.sun.jersey.contribs:jersey-apache-client4:jar</exclude> + <exclude>com.sun.jersey:jersey-client:jar</exclude> + <exclude>com.sun.jersey:jersey-core:jar</exclude> + <exclude>com.sun.jersey:jersey-server:jar</exclude> + <exclude>com.sun.xml.bind:jaxb-impl:jar</exclude> + <exclude>com.webank.wedatasphere.linkis:linkis-common:jar</exclude> + <exclude>commons-beanutils:commons-beanutils:jar</exclude> + <exclude>commons-beanutils:commons-beanutils-core:jar</exclude> + <exclude>commons-cli:commons-cli:jar</exclude> + <exclude>commons-collections:commons-collections:jar</exclude> + <exclude>commons-configuration:commons-configuration:jar</exclude> + <exclude>commons-daemon:commons-daemon:jar</exclude> + <exclude>commons-dbcp:commons-dbcp:jar</exclude> + <exclude>commons-digester:commons-digester:jar</exclude> + <exclude>commons-io:commons-io:jar</exclude> + <exclude>commons-lang:commons-lang:jar</exclude> + <exclude>commons-net:commons-net:jar</exclude> + <exclude>commons-pool:commons-pool:jar</exclude> + <exclude>io.netty:netty:jar</exclude> + <exclude>io.netty:netty-all:jar</exclude> + <exclude>io.netty:netty-buffer:jar</exclude> + <exclude>io.netty:netty-codec:jar</exclude> + <exclude>io.netty:netty-codec-http:jar</exclude> + <exclude>io.netty:netty-common:jar</exclude> + <exclude>io.netty:netty-handler:jar</exclude> + <exclude>io.netty:netty-transport:jar</exclude> + <exclude>io.netty:netty-transport-native-epoll:jar</exclude> + <exclude>io.reactivex:rxjava:jar</exclude> + <exclude>io.reactivex:rxnetty:jar</exclude> + <exclude>io.reactivex:rxnetty-contexts:jar</exclude> + <exclude>io.reactivex:rxnetty-servo:jar</exclude> + <exclude>javax.activation:activation:jar</exclude> + <exclude>javax.annotation:javax.annotation-api:jar</exclude> + <exclude>javax.inject:javax.inject:jar</exclude> + <exclude>javax.servlet:javax.servlet-api:jar</exclude> + <exclude>javax.servlet.jsp:jsp-api:jar</exclude> + <exclude>javax.xml.bind:jaxb-api:jar</exclude> + <exclude>javax.xml.stream:stax-api:jar</exclude> + <exclude>net.databinder.dispatch:dispatch-core_2.11:jar</exclude> + <exclude>net.databinder.dispatch:dispatch-json4s-jackson_2.11:jar</exclude> + <exclude>org.antlr:antlr-runtime:jar</exclude> + <exclude>org.antlr:stringtemplate:jar</exclude> + <exclude>org.apache.commons:commons-compress:jar</exclude> + <exclude>org.apache.commons:commons-math3:jar</exclude> + <exclude>org.apache.curator:curator-framework:jar</exclude> + <exclude>org.apache.curator:curator-recipes:jar</exclude> + <exclude>org.apache.directory.api:api-asn1-api:jar</exclude> + <exclude>org.apache.directory.api:api-util:jar</exclude> + <exclude>org.apache.directory.server:apacheds-i18n:jar</exclude> + <exclude>org.apache.directory.server:apacheds-kerberos-codec:jar</exclude> + <exclude>org.apache.hadoop:hadoop-annotations:jar</exclude> + <exclude>org.apache.hadoop:hadoop-auth:jar</exclude> + <exclude>org.apache.hadoop:hadoop-common:jar</exclude> + <exclude>org.apache.hadoop:hadoop-hdfs:jar</exclude> + <exclude>org.apache.htrace:htrace-core:jar</exclude> + <exclude>org.apache.logging.log4j:log4j-api:jar</exclude> + <exclude>org.apache.zookeeper:zookeeper:jar</exclude> + <exclude>org.aspectj:aspectjweaver:jar</exclude> + <exclude>org.bouncycastle:bcpkix-jdk15on:jar</exclude> + <exclude>org.bouncycastle:bcprov-jdk15on:jar</exclude> + <exclude>org.codehaus.jettison:jettison:jar</exclude> + <exclude>org.eclipse.jetty:jetty-continuation:jar</exclude> + <exclude>org.eclipse.jetty:jetty-http:jar</exclude> + <exclude>org.eclipse.jetty:jetty-io:jar</exclude> + <exclude>org.eclipse.jetty:jetty-jndi:jar</exclude> + <exclude>org.eclipse.jetty:jetty-plus:jar</exclude> + <exclude>org.eclipse.jetty:jetty-security:jar</exclude> + <exclude>org.eclipse.jetty:jetty-server:jar</exclude> + <exclude>org.eclipse.jetty:jetty-servlet:jar</exclude> + <exclude>org.eclipse.jetty:jetty-util:jar</exclude> + <exclude>org.eclipse.jetty:jetty-webapp:jar</exclude> + <exclude>org.eclipse.jetty:jetty-xml:jar</exclude> + <exclude>org.fusesource.leveldbjni:leveldbjni-all:jar</exclude> + <exclude>org.hdrhistogram:HdrHistogram:jar</exclude> + <exclude>org.json4s:json4s-ast_2.11:jar</exclude> + <exclude>org.json4s:json4s-core_2.11:jar</exclude> + <exclude>org.json4s:json4s-jackson_2.11:jar</exclude> + <exclude>org.jsoup:jsoup:jar</exclude> + <exclude>org.mortbay.jetty:jetty:jar</exclude> + <exclude>org.mortbay.jetty:jetty-util:jar</exclude> + <exclude>org.scala-lang.modules:scala-parser-combinators_2.11:jar</exclude> + <exclude>org.scala-lang.modules:scala-xml_2.11:jar</exclude> + <exclude>org.scala-lang:scala-compiler:jar</exclude> + <exclude>org.scala-lang:scala-library:jar</exclude> + <exclude>org.scala-lang:scala-reflect:jar</exclude> + <exclude>org.scala-lang:scalap:jar</exclude> + <exclude>org.slf4j:jul-to-slf4j:jar</exclude> + <exclude>org.slf4j:slf4j-api:jar</exclude> + <exclude>org.springframework.boot:spring-boot:jar</exclude> + <exclude>org.springframework.boot:spring-boot-autoconfigure:jar</exclude> + <exclude>org.springframework.boot:spring-boot-starter:jar</exclude> + <exclude>org.springframework.boot:spring-boot-starter-aop:jar</exclude> + <exclude>org.springframework.cloud:spring-cloud-commons:jar</exclude> + <exclude>org.springframework.cloud:spring-cloud-context:jar</exclude> + <exclude>org.springframework.cloud:spring-cloud-netflix-archaius:jar</exclude> + <exclude>org.springframework.cloud:spring-cloud-netflix-ribbon:jar</exclude> + <exclude>org.springframework.cloud:spring-cloud-starter:jar</exclude> + <exclude>org.springframework.cloud:spring-cloud-starter-netflix-archaius:jar</exclude> + <exclude>org.springframework.cloud:spring-cloud-starter-netflix-ribbon:jar</exclude> + <exclude>org.springframework.security:spring-security-crypto:jar</exclude> + <exclude>org.springframework.security:spring-security-rsa:jar</exclude> + <exclude>org.springframework:spring-aop:jar</exclude> + <exclude>org.springframework:spring-beans:jar</exclude> + <exclude>org.springframework:spring-context:jar</exclude> + <exclude>org.springframework:spring-core:jar</exclude> + <exclude>org.springframework:spring-expression:jar</exclude> + <!–<exclude>org.springframework:spring-jcl:jar</exclude>–> + <exclude>org.springframework:spring-web:jar</exclude> + <exclude>org.tukaani:xz:jar</exclude> + <exclude>org.yaml:snakeyaml:jar</exclude> + <exclude>xerces:xercesImpl:jar</exclude> + <exclude>xml-apis:xml-apis:jar</exclude> + <exclude>xmlenc:xmlenc:jar</exclude> + </excludes>--> + </dependencySet> + </dependencySets> + + <fileSets> + <fileSet> + <directory>${basedir}/conf</directory> + <includes> + <include>*</include> + </includes> + <fileMode>0777</fileMode> + <outputDirectory>conf</outputDirectory> + <lineEnding>unix</lineEnding> + </fileSet> + <fileSet> + <directory>${basedir}/bin</directory> + <includes> + <include>*</include> + </includes> + <fileMode>0777</fileMode> + <outputDirectory>bin</outputDirectory> + <lineEnding>unix</lineEnding> + </fileSet> + <fileSet> + <directory>.</directory> + <excludes> + <exclude>*/**</exclude> + </excludes> + <outputDirectory>logs</outputDirectory> + </fileSet> + </fileSets> + +</assembly> + diff --git a/datasource/metadatamanager/service/mysql/bin/start-mdm-mysql.sh b/datasource/metadatamanager/service/mysql/bin/start-mdm-mysql.sh new file mode 100644 index 0000000000..80cc775a4a --- /dev/null +++ b/datasource/metadatamanager/service/mysql/bin/start-mdm-mysql.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +cd `dirname $0` +cd .. +HOME=`pwd` + +export SERVER_PID=$HOME/bin/linkis.pid +export SERVER_LOG_PATH=$HOME/logs +export SERVER_CLASS=com.webank.wedatasphere.linkis.DataWorkCloudApplication + +if test -z "$SERVER_HEAP_SIZE" +then + export SERVER_HEAP_SIZE="512M" +fi + +if test -z "$SERVER_JAVA_OPTS" +then + export SERVER_JAVA_OPTS=" -Xmx$SERVER_HEAP_SIZE -XX:+UseG1GC -Xloggc:$HOME/logs/linkis-gc.log" +fi + +if [[ -f "${SERVER_PID}" ]]; then + pid=$(cat ${SERVER_PID}) + if kill -0 ${pid} >/dev/null 2>&1; then + echo "Server is already running." + exit 1 + fi +fi + +nohup java $SERVER_JAVA_OPTS -cp ../module/lib/*:$HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out & +pid=$! +if [[ -z "${pid}" ]]; then + echo "server $SERVER_NAME start failed!" + exit 1 +else + echo "server $SERVER_NAME start succeeded!" + echo $pid > $SERVER_PID + sleep 1 +fi \ No newline at end of file diff --git a/datasource/metadatamanager/service/mysql/bin/stop-mdm-mysql.sh b/datasource/metadatamanager/service/mysql/bin/stop-mdm-mysql.sh new file mode 100644 index 0000000000..f032887111 --- /dev/null +++ b/datasource/metadatamanager/service/mysql/bin/stop-mdm-mysql.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +cd `dirname $0` +cd .. +HOME=`pwd` + +export SERVER_PID=$HOME/bin/linkis.pid + +function wait_for_server_to_die() { + local pid + local count + pid=$1 + timeout=$2 + count=0 + timeoutTime=$(date "+%s") + let "timeoutTime+=$timeout" + currentTime=$(date "+%s") + forceKill=1 + + while [[ $currentTime -lt $timeoutTime ]]; do + $(kill ${pid} > /dev/null 2> /dev/null) + if kill -0 ${pid} > /dev/null 2>&1; then + sleep 3 + else + forceKill=0 + break + fi + currentTime=$(date "+%s") + done + + if [[ forceKill -ne 0 ]]; then + $(kill -9 ${pid} > /dev/null 2> /dev/null) + fi +} + +if [[ ! -f "${SERVER_PID}" ]]; then + echo "server $SERVER_NAME is not running" +else + pid=$(cat ${SERVER_PID}) + if [[ -z "${pid}" ]]; then + echo "server $SERVER_NAME is not running" + else + wait_for_server_to_die $pid 40 + $(rm -f ${SERVER_PID}) + echo "server $SERVER_NAME is stopped." + fi +fi \ No newline at end of file diff --git a/datasource/metadatamanager/service/hive/src/main/resources/application.yml b/datasource/metadatamanager/service/mysql/conf/application.yml similarity index 87% rename from datasource/metadatamanager/service/hive/src/main/resources/application.yml rename to datasource/metadatamanager/service/mysql/conf/application.yml index 1b52e767cd..bb31831572 100644 --- a/datasource/metadatamanager/service/hive/src/main/resources/application.yml +++ b/datasource/metadatamanager/service/mysql/conf/application.yml @@ -1,8 +1,8 @@ server: - port: 8293 + port: 8294 spring: application: - name: mdm-service-hive + name: mdm-service-mysql eureka: client: diff --git a/datasource/metadatamanager/service/hive/src/main/resources/linkis.properties b/datasource/metadatamanager/service/mysql/conf/linkis.properties similarity index 93% rename from datasource/metadatamanager/service/hive/src/main/resources/linkis.properties rename to datasource/metadatamanager/service/mysql/conf/linkis.properties index edea49cf37..f83c10d281 100644 --- a/datasource/metadatamanager/service/hive/src/main/resources/linkis.properties +++ b/datasource/metadatamanager/service/mysql/conf/linkis.properties @@ -19,6 +19,3 @@ wds.linkis.server.restful.scan.packages= #sit wds.linkis.server.version=v1 -#bml -wds.linkis.gateway.ip= -wds.linkis.gateway.port= \ No newline at end of file diff --git a/datasource/metadatamanager/server/src/main/resources/log4j.properties b/datasource/metadatamanager/service/mysql/conf/log4j.properties similarity index 100% rename from datasource/metadatamanager/server/src/main/resources/log4j.properties rename to datasource/metadatamanager/service/mysql/conf/log4j.properties diff --git a/datasource/metadatamanager/server/src/main/resources/log4j2.xml b/datasource/metadatamanager/service/mysql/conf/log4j2.xml similarity index 100% rename from datasource/metadatamanager/server/src/main/resources/log4j2.xml rename to datasource/metadatamanager/service/mysql/conf/log4j2.xml diff --git a/datasource/metadatamanager/service/mysql/pom.xml b/datasource/metadatamanager/service/mysql/pom.xml new file mode 100644 index 0000000000..1e671e4158 --- /dev/null +++ b/datasource/metadatamanager/service/mysql/pom.xml @@ -0,0 +1,124 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<!-- + ~ Copyright 2019 WeBank + ~ Licensed under the Apache License, Version 2.0 (the "License"); + ~ you may not use this file except in compliance with the License. + ~ You may obtain a copy of the License at + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <parent> + <artifactId>linkis</artifactId> + <groupId>com.webank.wedatasphere.linkis</groupId> + <version>0.9.4</version> + </parent> + <modelVersion>4.0.0</modelVersion> + + <artifactId>linkis-metadatamanager-service-mysql</artifactId> + + <properties> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <mysql.version>5.1.34</mysql.version> + </properties> + + <dependencies> + <dependency> + <groupId>com.webank.wedatasphere.linkis</groupId> + <artifactId>linkis-metadatamanager-common</artifactId> + <version>${linkis.version}</version> + </dependency> + <dependency> + <groupId>com.webank.wedatasphere.linkis</groupId> + <artifactId>linkis-module</artifactId> + <version>${linkis.version}</version> + <exclusions> + <exclusion> + <artifactId>asm</artifactId> + <groupId>org.ow2.asm</groupId> + </exclusion> + <exclusion> + <groupId>mysql</groupId> + <artifactId>mysql-connector-java</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>mysql</groupId> + <artifactId>mysql-connector-java</artifactId> + <version>${mysql.version}</version> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-deploy-plugin</artifactId> + </plugin> + + <plugin> + <groupId>net.alchim31.maven</groupId> + <artifactId>scala-maven-plugin</artifactId> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-assembly-plugin</artifactId> + <version>2.3</version> + <inherited>false</inherited> + <executions> + <execution> + <id>make-assembly</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + <configuration> + <descriptors> + <descriptor>src/main/assembly/distribution.xml</descriptor> + </descriptors> + </configuration> + </execution> + </executions> + <configuration> + <skipAssembly>false</skipAssembly> + <finalName>linkis-mdm-service-mysql</finalName> + <appendAssemblyId>false</appendAssemblyId> + <attach>false</attach> + <descriptors> + <descriptor>src/main/assembly/distribution.xml</descriptor> + </descriptors> + </configuration> + </plugin> + </plugins> + <resources> + <resource> + <directory>src/main/java</directory> + <includes> + <include>**/*.xml</include> + </includes> + </resource> + <resource> + <directory>src/main/resources</directory> + <excludes> + <exclude>**/*.properties</exclude> + <exclude>**/application.yml</exclude> + <exclude>**/bootstrap.yml</exclude> + <exclude>**/log4j2.xml</exclude> + </excludes> + </resource> + </resources> + <finalName>${project.artifactId}-${project.version}</finalName> + </build> +</project> diff --git a/datasource/metadatamanager/service/mysql/src/main/assembly/distribution.xml b/datasource/metadatamanager/service/mysql/src/main/assembly/distribution.xml new file mode 100644 index 0000000000..75dbe65ca6 --- /dev/null +++ b/datasource/metadatamanager/service/mysql/src/main/assembly/distribution.xml @@ -0,0 +1,206 @@ +<!-- + ~ Copyright 2019 WeBank + ~ + ~ Licensed under the Apache License, Version 2.0 (the "License"); + ~ you may not use this file except in compliance with the License. + ~ You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<assembly + xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/2.3"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/2.3 http://maven.apache.org/xsd/assembly-1.1.2.xsd";> + <id>linkis-mdm-service-mysql</id> + <formats> + <format>zip</format> + </formats> + <includeBaseDirectory>true</includeBaseDirectory> + <baseDirectory>linkis-mdm-service-mysql</baseDirectory> + + <dependencySets> + <dependencySet> + <!-- Enable access to all projects in the current multimodule build! <useAllReactorProjects>true</useAllReactorProjects> --> + <!-- Now, select which projects to include in this module-set. --> + <outputDirectory>lib</outputDirectory> + <useProjectArtifact>true</useProjectArtifact> + <useTransitiveDependencies>true</useTransitiveDependencies> + <unpack>false</unpack> + <useStrictFiltering>true</useStrictFiltering> + <useTransitiveFiltering>true</useTransitiveFiltering> + + <!-- <excludes> + <exclude>antlr:antlr:jar</exclude> + <exclude>aopalliance:aopalliance:jar</exclude> + <exclude>com.fasterxml.jackson.core:jackson-annotations:jar</exclude> + <exclude>com.fasterxml.jackson.core:jackson-core:jar</exclude> + <exclude>com.fasterxml.jackson.core:jackson-databind:jar</exclude> + <exclude>com.fasterxml.jackson.module:jackson-module-paranamer:jar</exclude> + <exclude>com.fasterxml.jackson.module:jackson-module-scala_2.11:jar</exclude> + <exclude>com.google.code.gson:gson:jar</exclude> + <exclude>com.google.guava:guava:jar</exclude> + <exclude>com.google.protobuf:protobuf-java:jar</exclude> + <exclude>com.netflix.archaius:archaius-core:jar</exclude> + <exclude>com.netflix.hystrix:hystrix-core:jar</exclude> + <exclude>com.netflix.netflix-commons:netflix-commons-util:jar</exclude> + <exclude>com.netflix.netflix-commons:netflix-statistics:jar</exclude> + <exclude>com.netflix.ribbon:ribbon:jar</exclude> + <exclude>com.netflix.ribbon:ribbon-core:jar</exclude> + <exclude>com.netflix.ribbon:ribbon-httpclient:jar</exclude> + <exclude>com.netflix.ribbon:ribbon-loadbalancer:jar</exclude> + <exclude>com.netflix.ribbon:ribbon-transport:jar</exclude> + <exclude>com.netflix.servo:servo-core:jar</exclude> + <exclude>com.sun.jersey.contribs:jersey-apache-client4:jar</exclude> + <exclude>com.sun.jersey:jersey-client:jar</exclude> + <exclude>com.sun.jersey:jersey-core:jar</exclude> + <exclude>com.sun.jersey:jersey-server:jar</exclude> + <exclude>com.sun.xml.bind:jaxb-impl:jar</exclude> + <exclude>com.webank.wedatasphere.linkis:linkis-common:jar</exclude> + <exclude>commons-beanutils:commons-beanutils:jar</exclude> + <exclude>commons-beanutils:commons-beanutils-core:jar</exclude> + <exclude>commons-cli:commons-cli:jar</exclude> + <exclude>commons-collections:commons-collections:jar</exclude> + <exclude>commons-configuration:commons-configuration:jar</exclude> + <exclude>commons-daemon:commons-daemon:jar</exclude> + <exclude>commons-dbcp:commons-dbcp:jar</exclude> + <exclude>commons-digester:commons-digester:jar</exclude> + <exclude>commons-io:commons-io:jar</exclude> + <exclude>commons-lang:commons-lang:jar</exclude> + <exclude>commons-net:commons-net:jar</exclude> + <exclude>commons-pool:commons-pool:jar</exclude> + <exclude>io.netty:netty:jar</exclude> + <exclude>io.netty:netty-all:jar</exclude> + <exclude>io.netty:netty-buffer:jar</exclude> + <exclude>io.netty:netty-codec:jar</exclude> + <exclude>io.netty:netty-codec-http:jar</exclude> + <exclude>io.netty:netty-common:jar</exclude> + <exclude>io.netty:netty-handler:jar</exclude> + <exclude>io.netty:netty-transport:jar</exclude> + <exclude>io.netty:netty-transport-native-epoll:jar</exclude> + <exclude>io.reactivex:rxjava:jar</exclude> + <exclude>io.reactivex:rxnetty:jar</exclude> + <exclude>io.reactivex:rxnetty-contexts:jar</exclude> + <exclude>io.reactivex:rxnetty-servo:jar</exclude> + <exclude>javax.activation:activation:jar</exclude> + <exclude>javax.annotation:javax.annotation-api:jar</exclude> + <exclude>javax.inject:javax.inject:jar</exclude> + <exclude>javax.servlet:javax.servlet-api:jar</exclude> + <exclude>javax.servlet.jsp:jsp-api:jar</exclude> + <exclude>javax.xml.bind:jaxb-api:jar</exclude> + <exclude>javax.xml.stream:stax-api:jar</exclude> + <exclude>net.databinder.dispatch:dispatch-core_2.11:jar</exclude> + <exclude>net.databinder.dispatch:dispatch-json4s-jackson_2.11:jar</exclude> + <exclude>org.antlr:antlr-runtime:jar</exclude> + <exclude>org.antlr:stringtemplate:jar</exclude> + <exclude>org.apache.commons:commons-compress:jar</exclude> + <exclude>org.apache.commons:commons-math3:jar</exclude> + <exclude>org.apache.curator:curator-framework:jar</exclude> + <exclude>org.apache.curator:curator-recipes:jar</exclude> + <exclude>org.apache.directory.api:api-asn1-api:jar</exclude> + <exclude>org.apache.directory.api:api-util:jar</exclude> + <exclude>org.apache.directory.server:apacheds-i18n:jar</exclude> + <exclude>org.apache.directory.server:apacheds-kerberos-codec:jar</exclude> + <exclude>org.apache.hadoop:hadoop-annotations:jar</exclude> + <exclude>org.apache.hadoop:hadoop-auth:jar</exclude> + <exclude>org.apache.hadoop:hadoop-common:jar</exclude> + <exclude>org.apache.hadoop:hadoop-hdfs:jar</exclude> + <exclude>org.apache.htrace:htrace-core:jar</exclude> + <exclude>org.apache.logging.log4j:log4j-api:jar</exclude> + <exclude>org.apache.zookeeper:zookeeper:jar</exclude> + <exclude>org.aspectj:aspectjweaver:jar</exclude> + <exclude>org.bouncycastle:bcpkix-jdk15on:jar</exclude> + <exclude>org.bouncycastle:bcprov-jdk15on:jar</exclude> + <exclude>org.codehaus.jettison:jettison:jar</exclude> + <exclude>org.eclipse.jetty:jetty-continuation:jar</exclude> + <exclude>org.eclipse.jetty:jetty-http:jar</exclude> + <exclude>org.eclipse.jetty:jetty-io:jar</exclude> + <exclude>org.eclipse.jetty:jetty-jndi:jar</exclude> + <exclude>org.eclipse.jetty:jetty-plus:jar</exclude> + <exclude>org.eclipse.jetty:jetty-security:jar</exclude> + <exclude>org.eclipse.jetty:jetty-server:jar</exclude> + <exclude>org.eclipse.jetty:jetty-servlet:jar</exclude> + <exclude>org.eclipse.jetty:jetty-util:jar</exclude> + <exclude>org.eclipse.jetty:jetty-webapp:jar</exclude> + <exclude>org.eclipse.jetty:jetty-xml:jar</exclude> + <exclude>org.fusesource.leveldbjni:leveldbjni-all:jar</exclude> + <exclude>org.hdrhistogram:HdrHistogram:jar</exclude> + <exclude>org.json4s:json4s-ast_2.11:jar</exclude> + <exclude>org.json4s:json4s-core_2.11:jar</exclude> + <exclude>org.json4s:json4s-jackson_2.11:jar</exclude> + <exclude>org.jsoup:jsoup:jar</exclude> + <exclude>org.mortbay.jetty:jetty:jar</exclude> + <exclude>org.mortbay.jetty:jetty-util:jar</exclude> + <exclude>org.scala-lang.modules:scala-parser-combinators_2.11:jar</exclude> + <exclude>org.scala-lang.modules:scala-xml_2.11:jar</exclude> + <exclude>org.scala-lang:scala-compiler:jar</exclude> + <exclude>org.scala-lang:scala-library:jar</exclude> + <exclude>org.scala-lang:scala-reflect:jar</exclude> + <exclude>org.scala-lang:scalap:jar</exclude> + <exclude>org.slf4j:jul-to-slf4j:jar</exclude> + <exclude>org.slf4j:slf4j-api:jar</exclude> + <exclude>org.springframework.boot:spring-boot:jar</exclude> + <exclude>org.springframework.boot:spring-boot-autoconfigure:jar</exclude> + <exclude>org.springframework.boot:spring-boot-starter:jar</exclude> + <exclude>org.springframework.boot:spring-boot-starter-aop:jar</exclude> + <exclude>org.springframework.cloud:spring-cloud-commons:jar</exclude> + <exclude>org.springframework.cloud:spring-cloud-context:jar</exclude> + <exclude>org.springframework.cloud:spring-cloud-netflix-archaius:jar</exclude> + <exclude>org.springframework.cloud:spring-cloud-netflix-ribbon:jar</exclude> + <exclude>org.springframework.cloud:spring-cloud-starter:jar</exclude> + <exclude>org.springframework.cloud:spring-cloud-starter-netflix-archaius:jar</exclude> + <exclude>org.springframework.cloud:spring-cloud-starter-netflix-ribbon:jar</exclude> + <exclude>org.springframework.security:spring-security-crypto:jar</exclude> + <exclude>org.springframework.security:spring-security-rsa:jar</exclude> + <exclude>org.springframework:spring-aop:jar</exclude> + <exclude>org.springframework:spring-beans:jar</exclude> + <exclude>org.springframework:spring-context:jar</exclude> + <exclude>org.springframework:spring-core:jar</exclude> + <exclude>org.springframework:spring-expression:jar</exclude> + <!–<exclude>org.springframework:spring-jcl:jar</exclude>–> + <exclude>org.springframework:spring-web:jar</exclude> + <exclude>org.tukaani:xz:jar</exclude> + <exclude>org.yaml:snakeyaml:jar</exclude> + <exclude>xerces:xercesImpl:jar</exclude> + <exclude>xml-apis:xml-apis:jar</exclude> + <exclude>xmlenc:xmlenc:jar</exclude> + </excludes>--> + </dependencySet> + </dependencySets> + + <fileSets> + <fileSet> + <directory>${basedir}/conf</directory> + <includes> + <include>*</include> + </includes> + <fileMode>0777</fileMode> + <outputDirectory>conf</outputDirectory> + <lineEnding>unix</lineEnding> + </fileSet> + <fileSet> + <directory>${basedir}/bin</directory> + <includes> + <include>*</include> + </includes> + <fileMode>0777</fileMode> + <outputDirectory>bin</outputDirectory> + <lineEnding>unix</lineEnding> + </fileSet> + <fileSet> + <directory>.</directory> + <excludes> + <exclude>*/**</exclude> + </excludes> + <outputDirectory>logs</outputDirectory> + </fileSet> + </fileSets> + +</assembly> + diff --git a/datasource/metadatamanager/service/mysql/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/SqlConnection.java b/datasource/metadatamanager/service/mysql/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/SqlConnection.java new file mode 100644 index 0000000000..383a0f1380 --- /dev/null +++ b/datasource/metadatamanager/service/mysql/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/SqlConnection.java @@ -0,0 +1,210 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.metadatamanager.service; + +import com.webank.wedatasphere.linkis.common.conf.CommonVars; +import com.webank.wedatasphere.linkis.metadatamanager.common.domain.MetaColumnInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.Closeable; +import java.io.IOException; +import java.sql.*; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * @author davidhua + * 2020/02/14 + */ +public class SqlConnection implements Closeable { + + private static final Logger LOG = LoggerFactory.getLogger(SqlConnection.class); + + private static final CommonVars<String> SQL_DRIVER_CLASS = + CommonVars.apply("wds.linkis.server.mdm.service.sql.driver", "com.mysql.jdbc.Driver"); + + private static final CommonVars<String> SQL_CONNECT_URL = + CommonVars.apply("wds.linkis.server.mdm.service.sql.url", "jdbc:mysql://%s:%s/%s"); + + private Connection conn; + + private ConnectMessage connectMessage; + + public SqlConnection(String host, Integer port, + String username, String password, + Map<String, Object> extraParams ) throws ClassNotFoundException, SQLException { + connectMessage = new ConnectMessage(host, port, username, password, extraParams); + conn = getDBConnection(connectMessage, ""); + //Try to create statement + Statement statement = conn.createStatement(); + statement.close(); + } + + public List<String> getAllDatabases() throws SQLException { + java.util.List<java.lang.String> dataBaseName = new ArrayList<>(); + Statement stmt = null; + ResultSet rs = null; + try{ + stmt = conn.createStatement(); + rs = stmt.executeQuery("SHOW DATABASES"); + while (rs.next()){ + dataBaseName.add(rs.getString(1)); + } + } finally { + closeResource(null, stmt, rs); + } + return dataBaseName; + } + + public List<String> getAllTables(String database) throws SQLException { + List<String> tableNames = new ArrayList<>(); + Statement stmt = null; + ResultSet rs = null; + try { + stmt = conn.createStatement(); + rs = stmt.executeQuery("SHOW TABLES FROM `" + database + "`"); + while (rs.next()) { + tableNames.add(rs.getString(1)); + } + return tableNames; + } finally{ + closeResource(null, stmt, rs); + } + } + + public List<MetaColumnInfo> getColumns(String database, String table) throws SQLException, ClassNotFoundException { + List<MetaColumnInfo> columns = new ArrayList<>(); + String columnSql = "SELECT * FROM `" + database +"`.`" + table + "` WHERE 1 = 2"; + PreparedStatement ps = null; + ResultSet rs = null; + ResultSetMetaData meta = null; + try { + List<String> primaryKeys = getPrimaryKeys(getDBConnection(connectMessage, database), table); + ps = conn.prepareStatement(columnSql); + rs = ps.executeQuery(); + meta = rs.getMetaData(); + int columnCount = meta.getColumnCount(); + for (int i = 1; i < columnCount + 1; i++) { + MetaColumnInfo info = new MetaColumnInfo(); + info.setIndex(i); + info.setName(meta.getColumnName(i)); + info.setType(meta.getColumnTypeName(i)); + if(primaryKeys.contains(meta.getColumnName(i))){ + info.setPrimaryKey(true); + } + columns.add(info); + } + }finally { + closeResource(null, ps, rs); + } + return columns; + } + + /** + * Get primary keys + * @param connection connection + * @param table table name + * @return + * @throws SQLException + */ + private List<String> getPrimaryKeys(Connection connection, String table) throws SQLException { + ResultSet rs = null; + List<String> primaryKeys = new ArrayList<>(); + try { + DatabaseMetaData dbMeta = connection.getMetaData(); + rs = dbMeta.getPrimaryKeys(null, null, table); + while(rs.next()){ + primaryKeys.add(rs.getString("column_name")); + } + return primaryKeys; + }finally{ + if(null != rs){ + closeResource(connection, null, rs); + } + } + } + + /** + * close database resource + * @param connection connection + * @param statement statement + * @param resultSet result set + */ + private void closeResource(Connection connection, Statement statement, ResultSet resultSet){ + try { + if(null != resultSet && !resultSet.isClosed()) { + resultSet.close(); + } + if(null != statement && !statement.isClosed()){ + statement.close(); + } + if(null != connection && !connection.isClosed()){ + connection.close(); + } + }catch (SQLException e){ + LOG.warn("Fail to release resource [" + e.getMessage() +"]", e); + } + } + + @Override + public void close() throws IOException { + closeResource(conn, null, null); + } + + /** + * @param connectMessage + * @param database + * @return + * @throws ClassNotFoundException + */ + private Connection getDBConnection(ConnectMessage connectMessage, String database) throws ClassNotFoundException, SQLException { + String extraParamString = connectMessage.extraParams.entrySet() + .stream().map(e -> String.join("=", e.getKey(), String.valueOf(e.getValue()))) + .collect(Collectors.joining("&")); + Class.forName(SQL_DRIVER_CLASS.getValue()); + String url = String.format(SQL_CONNECT_URL.getValue(), connectMessage.host, connectMessage.port, database); + if(!connectMessage.extraParams.isEmpty()) { + url += "?" + extraParamString; + } + return DriverManager.getConnection(url, connectMessage.username, connectMessage.password); + } + + /** + * Connect message + */ + private static class ConnectMessage{ + private String host; + + private Integer port; + + private String username; + + private String password; + + private Map<String, Object> extraParams; + + public ConnectMessage(String host, Integer port, + String username, String password, + Map<String, Object> extraParams){ + this.host = host; + this.port = port; + this.username = username; + this.password = password; + this.extraParams = extraParams; + } + } +} diff --git a/datasource/metadatamanager/service/mysql/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/SqlMetaService.java b/datasource/metadatamanager/service/mysql/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/SqlMetaService.java new file mode 100644 index 0000000000..744ee4a051 --- /dev/null +++ b/datasource/metadatamanager/service/mysql/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/SqlMetaService.java @@ -0,0 +1,79 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.metadatamanager.service; + +import com.webank.wedatasphere.linkis.metadatamanager.common.Json; +import com.webank.wedatasphere.linkis.metadatamanager.common.domain.MetaColumnInfo; +import com.webank.wedatasphere.linkis.metadatamanager.common.service.AbstractMetaService; +import com.webank.wedatasphere.linkis.metadatamanager.common.service.MetadataConnection; +import org.springframework.stereotype.Component; + +import java.sql.SQLException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * @author davidhua + * 2020/02/14 + */ +@Component +public class SqlMetaService extends AbstractMetaService<SqlConnection> { + @Override + public MetadataConnection<SqlConnection> getConnection(String operator, Map<String, Object> params) throws Exception { + String host = String.valueOf(params.getOrDefault(SqlParamsMapper.PARAM_SQL_HOST.getValue(), "")); + //After deserialize, Integer will be Double, Why? + Integer port = (Double.valueOf(String.valueOf(params.getOrDefault(SqlParamsMapper.PARAM_SQL_PORT.getValue(), 0)))).intValue(); + String username = String.valueOf(params.getOrDefault(SqlParamsMapper.PARAM_SQL_USERNAME.getValue(), "")); + String password = String.valueOf(params.getOrDefault(SqlParamsMapper.PARAM_SQL_PASSWORD.getValue(), "")); + Map<String, Object> extraParams = new HashMap<>(); + Object sqlParamObj = params.get(SqlParamsMapper.PARAM_SQL_EXTRA_PARAMS.getValue()); + if(null != sqlParamObj){ + if(!(sqlParamObj instanceof Map)){ + extraParams = Json.fromJson(String.valueOf(sqlParamObj), Map.class, String.class, Object.class); + }else{ + extraParams = (Map<String, Object>)sqlParamObj; + } + } + assert extraParams != null; + return new MetadataConnection<>(new SqlConnection(host, port, username, password, extraParams)); + } + + @Override + public List<String> queryDatabases(SqlConnection connection) { + try { + return connection.getAllDatabases(); + } catch (SQLException e) { + throw new RuntimeException("Fail to get Sql databases(获取数据库列表失败)", e); + } + } + + @Override + public List<String> queryTables(SqlConnection connection, String database) { + try { + return connection.getAllTables(database); + } catch (SQLException e) { + throw new RuntimeException("Fail to get Sql tables(获取表列表失败)", e); + } + } + + @Override + public List<MetaColumnInfo> queryColumns(SqlConnection connection, String database, String table) { + try { + return connection.getColumns(database, table); + } catch (SQLException | ClassNotFoundException e) { + throw new RuntimeException("Fail to get Sql columns(获取字段列表失败)", e); + } + } +} diff --git a/datasource/metadatamanager/service/mysql/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/SqlParamsMapper.java b/datasource/metadatamanager/service/mysql/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/SqlParamsMapper.java new file mode 100644 index 0000000000..c41285e4e0 --- /dev/null +++ b/datasource/metadatamanager/service/mysql/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/SqlParamsMapper.java @@ -0,0 +1,38 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.metadatamanager.service; + +import com.webank.wedatasphere.linkis.common.conf.CommonVars; + +/** + * @author davidhua + * 2020/02/14 + */ +public class SqlParamsMapper { + + public static final CommonVars<String> PARAM_SQL_HOST = + CommonVars.apply("wds.linkis.server.mdm.service.sql.host", "host"); + + public static final CommonVars<String> PARAM_SQL_PORT = + CommonVars.apply("wds.linkis.server.mdm.service.sql.port", "port"); + + public static final CommonVars<String> PARAM_SQL_USERNAME = + CommonVars.apply("wds.linkis.server.mdm.service.sql.username", "username"); + + public static final CommonVars<String> PARAM_SQL_PASSWORD = + CommonVars.apply("wds.linkis.server.mdm.service.sql.password", "password"); + + public static final CommonVars<String> PARAM_SQL_EXTRA_PARAMS = + CommonVars.apply("wds.linkis.server.mdm.service.sql.params", "params"); +} diff --git a/datasource/metadatamanager/service/mysql/src/main/scala/com/webank/wedatasphere/linkis/metadatamanager/service/receiver/SqlReceiver.scala b/datasource/metadatamanager/service/mysql/src/main/scala/com/webank/wedatasphere/linkis/metadatamanager/service/receiver/SqlReceiver.scala new file mode 100644 index 0000000000..11f24a9ee9 --- /dev/null +++ b/datasource/metadatamanager/service/mysql/src/main/scala/com/webank/wedatasphere/linkis/metadatamanager/service/receiver/SqlReceiver.scala @@ -0,0 +1,28 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.metadatamanager.service.receiver + +import com.webank.wedatasphere.linkis.DataWorkCloudApplication +import com.webank.wedatasphere.linkis.metadatamanager.common.receiver.BaseMetaReceiver +import com.webank.wedatasphere.linkis.metadatamanager.common.service.MetadataService +import javax.annotation.PostConstruct +import org.springframework.stereotype.Component + +@Component +class SqlReceiver extends BaseMetaReceiver{ + @PostConstruct + def init(): Unit = { + metadataService = DataWorkCloudApplication.getApplicationContext.getBean(classOf[MetadataService]) + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
