This is an automated email from the ASF dual-hosted git repository. aiceflower pushed a commit to branch release-0.9.4 in repository https://gitbox.apache.org/repos/asf/linkis.git
commit 5971c5e1cc62097d2d7b7f442812c32b920d3581 Author: zhengfan199525 <[email protected]> AuthorDate: Thu May 7 13:23:11 2020 +0800 add metadatamanager/hive --- datasource/metadatamanager/service/hive/pom.xml | 117 ++++++++++++ .../metadatamanager/service/HiveConnection.java | 139 +++++++++++++++ .../metadatamanager/service/HiveMetaService.java | 197 +++++++++++++++++++++ .../metadatamanager/service/HiveParamsMapper.java | 33 ++++ .../hive/src/main/resources/application.yml | 23 +++ .../hive/src/main/resources/linkis.properties | 24 +++ .../hive/src/main/resources/log4j.properties | 33 ++++ .../service/hive/src/main/resources/log4j2.xml | 35 ++++ .../service/receiver/HiveReceiver.scala | 29 +++ 9 files changed, 630 insertions(+) diff --git a/datasource/metadatamanager/service/hive/pom.xml b/datasource/metadatamanager/service/hive/pom.xml new file mode 100644 index 0000000000..3f28e117f6 --- /dev/null +++ b/datasource/metadatamanager/service/hive/pom.xml @@ -0,0 +1,117 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<!-- + ~ Copyright 2019 WeBank + ~ Licensed under the Apache License, Version 2.0 (the "License"); + ~ you may not use this file except in compliance with the License. + ~ You may obtain a copy of the License at + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <parent> + <artifactId>linkis</artifactId> + <groupId>com.webank.wedatasphere.linkis</groupId> + <version>0.9.3</version> + </parent> + <modelVersion>4.0.0</modelVersion> + <artifactId>linkis-metadatamanager-service-hive</artifactId> + + <properties> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <hive.version>1.2.1</hive.version> + <hadoop.version>2.7.2</hadoop.version> + </properties> + + <dependencies> + <dependency> + <groupId>com.webank.wedatasphere.linkis</groupId> + <artifactId>linkis-metadatamanager-common</artifactId> + <version>${linkis.version}</version> + </dependency> + <dependency> + <groupId>com.webank.wedatasphere.linkis</groupId> + <artifactId>linkis-module</artifactId> + <version>${linkis.version}</version> + </dependency> + <!--Hive dependencies--> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-client</artifactId> + <version>${hadoop.version}</version> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + <exclusion> + <artifactId>servlet-api</artifactId> + <groupId>javax.servlet</groupId> + </exclusion> + <exclusion> + <artifactId>guava</artifactId> + <groupId>com.google.guava</groupId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-exec</artifactId> + <version>${hive.version}</version> + <exclusions> + <exclusion> + <artifactId>slf4j-log4j12</artifactId> + <groupId>org.slf4j</groupId> + </exclusion> + <exclusion> + <groupId>org.apache.ivy</groupId> + <artifactId>ivy</artifactId> + </exclusion> + <exclusion> + <artifactId>guava</artifactId> + <groupId>com.google.guava</groupId> + </exclusion> + <exclusion> + <artifactId>commons-lang3</artifactId> + <groupId>org.apache.commons</groupId> + </exclusion> + </exclusions> + </dependency> + <!--bml client--> + <dependency> + <groupId>com.webank.wedatasphere.linkis</groupId> + <artifactId>linkis-bmlclient</artifactId> + <version>${linkis.version}</version> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-deploy-plugin</artifactId> + </plugin> + + <plugin> + <groupId>net.alchim31.maven</groupId> + <artifactId>scala-maven-plugin</artifactId> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + </plugin> + </plugins> + <resources> + <resource> + <directory>src/main/resources</directory> + </resource> + </resources> + <finalName>${project.artifactId}-${project.version}</finalName> + </build> +</project> diff --git a/datasource/metadatamanager/service/hive/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/HiveConnection.java b/datasource/metadatamanager/service/hive/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/HiveConnection.java new file mode 100644 index 0000000000..f581bdc1ba --- /dev/null +++ b/datasource/metadatamanager/service/hive/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/HiveConnection.java @@ -0,0 +1,139 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.metadatamanager.service; + +import com.webank.wedatasphere.linkis.common.conf.CommonVars; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.ql.metadata.Hive; +import org.apache.hadoop.security.UserGroupInformation; + +import java.io.Closeable; +import java.io.IOException; +import java.net.URI; +import java.security.PrivilegedExceptionAction; +import java.util.concurrent.locks.ReentrantLock; + +import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY; + +/** + * @author davidhua + * 2020/02/14 + */ +public class HiveConnection implements Closeable { + + private Hive hiveClient; + + private IMetaStoreClient metaStoreClient; + + private static final CommonVars<String> KERBEROS_DEFAULT_PRINCIPLE = + CommonVars.apply("wds.linkis.server.mdm.service.kerberos.principle", "hadoop/[email protected]"); + + private static final CommonVars<String> DEFAULT_SERVICE_USER = + CommonVars.apply("wds.linkis.server.mdm.service.user", "hadoop"); + + private static final CommonVars<String> KERBEROS_KRB5_CONF_PATH = + CommonVars.apply("wds.linkis.server.mdm.service.kerberos.krb5.path", ""); + + static{ + if(StringUtils.isNotBlank(KERBEROS_KRB5_CONF_PATH.getValue())){ + System.setProperty("java.security.krb5.conf", KERBEROS_KRB5_CONF_PATH.getValue()); + } + } + + public HiveConnection(String uris, String principle, String keytabFilePath) throws Exception { + final HiveConf conf = new HiveConf(); + conf.setVar(HiveConf.ConfVars.METASTOREURIS, uris); + conf.setVar(HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL, "true"); + conf.setVar(HiveConf.ConfVars.METASTORE_KERBEROS_PRINCIPAL, KERBEROS_DEFAULT_PRINCIPLE.getValue()); + //Disable the cache in FileSystem + conf.setBoolean(String.format("fs.%s.impl.disable.cache", URI.create(conf.get(FS_DEFAULT_NAME_KEY, "")).getScheme()), true); + conf.set("hadoop.security.authentication", "kerberos"); + principle = principle.substring(0, principle.indexOf("@")); + UserGroupInformation ugi = UserGroupInformationWrapper.loginUserFromKeytab(conf, + principle, keytabFilePath); + hiveClient = getHive(ugi, conf); + } + + public HiveConnection(String uris) throws Exception{ + final HiveConf conf = new HiveConf(); + conf.setVar(HiveConf.ConfVars.METASTOREURIS, uris); + //Disable the cache in FileSystem + conf.setBoolean(String.format("fs.%s.impl.disable.cache", URI.create(conf.get(FS_DEFAULT_NAME_KEY, "")).getScheme()), true); + //TODO choose an authentication strategy for hive, and then use createProxyUser + UserGroupInformation ugi = UserGroupInformation.createRemoteUser(DEFAULT_SERVICE_USER.getValue()); + hiveClient = getHive(ugi, conf); + } + /** + * Get Hive client(Hive object) + * @return hive + */ + public Hive getClient(){ + return hiveClient; + } + + private Hive getHive(UserGroupInformation ugi, HiveConf conf) throws IOException, InterruptedException { + return ugi.doAs((PrivilegedExceptionAction<Hive>) () -> { + Hive hive = Hive.get(conf); + metaStoreClient = hive.getMSC(); + //To remove thread Local vars + Hive.set(null); + return hive; + }); + } + @Override + public void close() throws IOException { + //Close meta store client + metaStoreClient.close(); + } + + /** + * Wrapper class of UserGroupInformation + */ + private static class UserGroupInformationWrapper{ + private static ReentrantLock globalLock = new ReentrantLock(); + + public static UserGroupInformation loginUserFromKeytab(final Configuration conf, String user, String path) throws Exception { + globalLock.lock(); + try{ + UserGroupInformation.setConfiguration(conf); + return UserGroupInformation.loginUserFromKeytabAndReturnUGI(user, path); + }finally{ + globalLock.unlock(); + } + } + public static UserGroupInformation createProxyUser(final Configuration conf, String user) throws Exception{ + globalLock.lock(); + try{ + UserGroupInformation.setLoginUser(null); + UserGroupInformation.setConfiguration(conf); + return UserGroupInformation.createProxyUser(user, UserGroupInformation.getLoginUser()); + }finally{ + globalLock.unlock(); + } + } + + public static UserGroupInformation getLoginUser() throws Exception{ + globalLock.lock(); + try{ + UserGroupInformation.setLoginUser(null); + return UserGroupInformation.getLoginUser(); + }finally{ + globalLock.unlock(); + } + } + } +} diff --git a/datasource/metadatamanager/service/hive/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/HiveMetaService.java b/datasource/metadatamanager/service/hive/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/HiveMetaService.java new file mode 100644 index 0000000000..4ea3581998 --- /dev/null +++ b/datasource/metadatamanager/service/hive/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/HiveMetaService.java @@ -0,0 +1,197 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.metadatamanager.service; + +import com.webank.wedatasphere.linkis.bml.client.BmlClient; +import com.webank.wedatasphere.linkis.bml.client.BmlClientFactory; +import com.webank.wedatasphere.linkis.bml.protocol.BmlDownloadResponse; +import com.webank.wedatasphere.linkis.common.conf.CommonVars; +import com.webank.wedatasphere.linkis.metadatamanager.common.domain.MetaColumnInfo; +import com.webank.wedatasphere.linkis.metadatamanager.common.domain.MetaPartitionInfo; +import com.webank.wedatasphere.linkis.metadatamanager.common.exception.MetaRuntimeException; +import com.webank.wedatasphere.linkis.metadatamanager.common.service.AbstractMetaService; +import com.webank.wedatasphere.linkis.metadatamanager.common.service.MetadataConnection; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.core.io.Resource; +import org.springframework.core.io.support.PathMatchingResourcePatternResolver; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.*; + +/** + * @author davidhua + * 2020/02/14 + */ +@Component +public class HiveMetaService extends AbstractMetaService<HiveConnection> { + + private static final Logger LOG = LoggerFactory.getLogger(HiveMetaService.class); + private static final CommonVars<String> TMP_FILE_STORE_LOCATION = + CommonVars.apply("wds.linkis.server.mdm.service.temp.location", "classpath:/tmp"); + + private BmlClient client; + + @PostConstruct + public void buildClient(){ + client = BmlClientFactory.createBmlClient(); + } + @Override + public MetadataConnection<HiveConnection> getConnection(String operator, Map<String, Object> params) throws Exception { + Resource resource = new PathMatchingResourcePatternResolver().getResource(TMP_FILE_STORE_LOCATION.getValue()); + String uris = String.valueOf(params.getOrDefault(HiveParamsMapper.PARAM_HIVE_URIS.getValue(), "")); + String principle = String.valueOf(params.getOrDefault(HiveParamsMapper.PARAM_HIVE_PRINCIPLE.getValue(), "")); + HiveConnection conn = null; + if(StringUtils.isNotBlank(principle)){ + LOG.info("Try to connect Hive MetaStore in kerberos with principle:[" + principle +"]"); + String keytabResourceId = String.valueOf(params.getOrDefault(HiveParamsMapper.PARAM_HIVE_KEYTAB.getValue(), "")); + if(StringUtils.isNotBlank(keytabResourceId)){ + LOG.info("Start to download resource id:[" + keytabResourceId +"]"); + String keytabFilePath = resource.getFile().getAbsolutePath() + "/" + UUID.randomUUID().toString().replace("-", ""); + if(!downloadResource(keytabResourceId, operator, keytabFilePath)){ + throw new MetaRuntimeException("Fail to download resource i:[" + keytabResourceId +"]"); + } + conn = new HiveConnection(uris, principle, keytabFilePath); + }else{ + throw new MetaRuntimeException("Cannot find the keytab file in connect parameters"); + } + }else{ + conn = new HiveConnection(uris); + } + return new MetadataConnection<>(conn, true); + } + + /** + * Download resource to path by id + * @param resourceId resource id + * @param user user + * @param absolutePath absolute path + * @return + * @throws IOException + */ + private boolean downloadResource(String resourceId, String user, String absolutePath) throws IOException { + BmlDownloadResponse downloadResponse = client.downloadResource(user, resourceId, absolutePath); + if(downloadResponse.isSuccess()){ + IOUtils.copy(downloadResponse.inputStream(), new FileOutputStream(absolutePath)); + return true; + } + return false; + } + + @Override + public List<String> queryDatabases(HiveConnection connection) { + try { + return connection.getClient().getAllDatabases(); + } catch (HiveException e) { + throw new RuntimeException("Fail to get Hive databases(获取数据库列表失败)", e); + } + } + + @Override + public List<String> queryTables(HiveConnection connection, String database) { + try { + return connection.getClient().getAllTables(database); + } catch (HiveException e) { + throw new RuntimeException("Fail to get Hive tables(获取表列表失败)", e); + } + } + + @Override + public MetaPartitionInfo queryPartitions(HiveConnection connection, String database, String table) { + List<Partition> partitions; + Table rawTable; + try { + rawTable = connection.getClient().getTable(database, table); + partitions = connection.getClient().getPartitions(rawTable); + } catch (HiveException e) { + throw new RuntimeException("Fail to get Hive partitions(获取分区信息失败)", e); + } + MetaPartitionInfo info = new MetaPartitionInfo(); + List<FieldSchema> partitionKeys = rawTable.getPartitionKeys(); + List<String> partKeys = new ArrayList<>(); + partitionKeys.forEach(e -> partKeys.add(e.getName())); + info.setPartKeys(partKeys); + //Static partitions + Map<String, MetaPartitionInfo.PartitionNode> pMap = new HashMap<>(20); + MetaPartitionInfo.PartitionNode root = new MetaPartitionInfo.PartitionNode(); + info.setRoot(root); + long t = System.currentTimeMillis(); + for(Partition p : partitions){ + try { + List<String> values = p.getValues(); + if(!partitionKeys.isEmpty()){ + String parentNameValue = ""; + for(int i = 0; i < values.size(); i++){ + FieldSchema fieldSchema = partitionKeys.get(i); + String name = fieldSchema.getName(); + String value = values.get(i); + String nameValue= name + "=" + value; + MetaPartitionInfo.PartitionNode node = new MetaPartitionInfo.PartitionNode(); + if(i > 0){ + MetaPartitionInfo.PartitionNode parent = pMap.get(parentNameValue); + parent.setName(name); + parent.getPartitions().putIfAbsent(value, node); + }else{ + root.setName(name); + root.getPartitions().putIfAbsent(value, node); + } + parentNameValue += "/" + nameValue; + pMap.putIfAbsent(parentNameValue, node); + } + } + }catch(Exception e){ + LOG.warn(e.getMessage(), e); + } + } + return info; + } + + @Override + public List<MetaColumnInfo> queryColumns(HiveConnection connection, String database, String table) { + List<MetaColumnInfo> columns = new ArrayList<>(); + Table tb; + try { + tb = connection.getClient().getTable(database, table); + } catch (HiveException e) { + throw new RuntimeException("Fail to get Hive columns(获得表字段信息失败)", e); + } + tb.getFields().forEach( field ->{ + MetaColumnInfo metaColumnInfo = new MetaColumnInfo(); + metaColumnInfo.setIndex(field.getFieldID()); + metaColumnInfo.setName(field.getFieldName()); + metaColumnInfo.setType(field.getFieldObjectInspector().getTypeName()); + columns.add(metaColumnInfo); + }); + return columns; + } + + @Override + public Map<String, String> queryTableProps(HiveConnection connection, String database, String table) { + try { + Table rawTable = connection.getClient().getTable(database, table); + return new HashMap<>((Map)rawTable.getMetadata()); + }catch(Exception e){ + throw new RuntimeException("Fail to get Hive table properties(获取表参数信息失败)", e); + } + } +} diff --git a/datasource/metadatamanager/service/hive/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/HiveParamsMapper.java b/datasource/metadatamanager/service/hive/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/HiveParamsMapper.java new file mode 100644 index 0000000000..6a3372b293 --- /dev/null +++ b/datasource/metadatamanager/service/hive/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/HiveParamsMapper.java @@ -0,0 +1,33 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.metadatamanager.service; + +import com.webank.wedatasphere.linkis.common.conf.CommonVars; + +/** + * @author davidhua + * 2020/02/14 + */ +public class HiveParamsMapper { + + public static final CommonVars<String> PARAM_HIVE_PRINCIPLE = + CommonVars.apply("wds.linkis.server.mdm.service.hive.principle", "principle"); + + public static final CommonVars<String> PARAM_HIVE_URIS = + CommonVars.apply("wds.linkis.server.mdm.service.hive.uris", "uris"); + + public static final CommonVars<String> PARAM_HIVE_KEYTAB = + CommonVars.apply("wds.linkis.server.mdm.service.hive.keytab", "keytab"); + +} diff --git a/datasource/metadatamanager/service/hive/src/main/resources/application.yml b/datasource/metadatamanager/service/hive/src/main/resources/application.yml new file mode 100644 index 0000000000..1b52e767cd --- /dev/null +++ b/datasource/metadatamanager/service/hive/src/main/resources/application.yml @@ -0,0 +1,23 @@ +server: + port: 8293 +spring: + application: + name: mdm-service-hive + +eureka: + client: + serviceUrl: + defaultZone: http://${ip}:${port}/eureka/ + instance: + metadata-map: + test: wedatasphere + +management: + endpoints: + web: + exposure: + include: refresh,info +logging: + config: classpath:log4j2.xml + + diff --git a/datasource/metadatamanager/service/hive/src/main/resources/linkis.properties b/datasource/metadatamanager/service/hive/src/main/resources/linkis.properties new file mode 100644 index 0000000000..edea49cf37 --- /dev/null +++ b/datasource/metadatamanager/service/hive/src/main/resources/linkis.properties @@ -0,0 +1,24 @@ +# +# Copyright 2019 WeBank +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +wds.linkis.server.mybatis.mapperLocations= +wds.linkis.server.mybatis.typeAliasesPackage= +wds.linkis.server.mybatis.BasePackage= +wds.linkis.server.restful.scan.packages= + +#sit +wds.linkis.server.version=v1 + +#bml +wds.linkis.gateway.ip= +wds.linkis.gateway.port= \ No newline at end of file diff --git a/datasource/metadatamanager/service/hive/src/main/resources/log4j.properties b/datasource/metadatamanager/service/hive/src/main/resources/log4j.properties new file mode 100644 index 0000000000..d5ee44b86b --- /dev/null +++ b/datasource/metadatamanager/service/hive/src/main/resources/log4j.properties @@ -0,0 +1,33 @@ +# +# Copyright 2019 WeBank +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +### set log levels ### + +log4j.rootCategory=INFO,console + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.Threshold=INFO +log4j.appender.console.layout=org.apache.log4j.PatternLayout +#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n +log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n + + +log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender +log4j.appender.com.webank.bdp.ide.core.Threshold=INFO +log4j.additivity.com.webank.bdp.ide.core=false +log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout +log4j.appender.com.webank.bdp.ide.core.Append=true +log4j.appender.com.webank.bdp.ide.core.File=logs/linkis.log +log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n + +log4j.logger.org.springframework=INFO \ No newline at end of file diff --git a/datasource/metadatamanager/service/hive/src/main/resources/log4j2.xml b/datasource/metadatamanager/service/hive/src/main/resources/log4j2.xml new file mode 100644 index 0000000000..1c68190669 --- /dev/null +++ b/datasource/metadatamanager/service/hive/src/main/resources/log4j2.xml @@ -0,0 +1,35 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Copyright 2019 WeBank + ~ Licensed under the Apache License, Version 2.0 (the "License"); + ~ you may not use this file except in compliance with the License. + ~ You may obtain a copy of the License at + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<configuration status="error" monitorInterval="30"> + <appenders> + <Console name="Console" target="SYSTEM_OUT"> + <ThresholdFilter level="INFO" onMatch="ACCEPT" onMismatch="DENY"/> + <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} %-5level [%t] %logger{36} %L %M - %msg%xEx%n"/> + </Console> + <RollingFile name="RollingFile" fileName="logs/linkis.log" + filePattern="logs/$${date:yyyy-MM}/linkis-log-%d{yyyy-MM-dd}-%i.log"> + <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} [%-5level] [%-40t] %c{1.} (%L) [%M] - %msg%xEx%n"/> + <SizeBasedTriggeringPolicy size="100MB"/> + <DefaultRolloverStrategy max="20"/> + </RollingFile> + </appenders> + <loggers> + <root level="INFO"> + <appender-ref ref="RollingFile"/> + <appender-ref ref="Console"/> + </root> + </loggers> +</configuration> + diff --git a/datasource/metadatamanager/service/hive/src/main/scala/com/webank/wedatasphere/linkis/metadatamanager/service/receiver/HiveReceiver.scala b/datasource/metadatamanager/service/hive/src/main/scala/com/webank/wedatasphere/linkis/metadatamanager/service/receiver/HiveReceiver.scala new file mode 100644 index 0000000000..9d139155c7 --- /dev/null +++ b/datasource/metadatamanager/service/hive/src/main/scala/com/webank/wedatasphere/linkis/metadatamanager/service/receiver/HiveReceiver.scala @@ -0,0 +1,29 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.metadatamanager.service.receiver + +import com.webank.wedatasphere.linkis.DataWorkCloudApplication +import com.webank.wedatasphere.linkis.metadatamanager.common.service.MetadataService +import com.webank.wedatasphere.linkis.metadatamanager.common.receiver.BaseMetaReceiver +import com.webank.wedatasphere.linkis.metadatamanager.common.service.MetadataService +import javax.annotation.PostConstruct +import org.springframework.stereotype.Component + +@Component +class HiveReceiver extends BaseMetaReceiver{ + @PostConstruct + def init(): Unit = { + metadataService = DataWorkCloudApplication.getApplicationContext.getBean(classOf[MetadataService]) + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
