test thrift metastore client
Project: http://git-wip-us.apache.org/repos/asf/incubator-griffin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-griffin/commit/5a7793d1 Tree: http://git-wip-us.apache.org/repos/asf/incubator-griffin/tree/5a7793d1 Diff: http://git-wip-us.apache.org/repos/asf/incubator-griffin/diff/5a7793d1 Branch: refs/heads/griffin-0.2.0-incubating-rc4 Commit: 5a7793d1d957f07dda9ad9dd3e8c8e85ac26ddd4 Parents: 41e986b Author: Lionel Liu <[email protected]> Authored: Tue Apr 24 08:41:14 2018 +0800 Committer: Lionel Liu <[email protected]> Committed: Tue Apr 24 08:41:14 2018 +0800 ---------------------------------------------------------------------- service/pom.xml | 148 +++++++++++++++++-- .../core/metastore/hive/HiveMetaStoreProxy.java | 96 +++++++++++- .../hive/HiveMetaStoreServiceImpl.java | 37 ++++- 3 files changed, 253 insertions(+), 28 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/5a7793d1/service/pom.xml ---------------------------------------------------------------------- diff --git a/service/pom.xml b/service/pom.xml index 9c8f3bf..8105bb3 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -36,7 +36,7 @@ under the License. <java.version>1.8</java.version> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <hadoop.version>2.7.1</hadoop.version> - <hive.version>2.2.0</hive.version> + <hive.version>2.3.3</hive.version> <scala.version>2.10</scala.version> <spring.boot.version>1.5.1.RELEASE</spring.boot.version> <confluent.version>3.2.0</confluent.version> @@ -70,10 +70,38 @@ under the License. </dependencyManagement> <dependencies> + <!--<dependency>--> + <!--<groupId>javax.servlet</groupId>--> + <!--<artifactId>javax.servlet-api</artifactId>--> + <!--<version>3.1.0</version>--> + <!--</dependency>--> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-web</artifactId> + <exclusions> + <!--<exclusion>--> + <!--<groupId>org.springframework.boot</groupId>--> + <!--<artifactId>spring-boot-starter-tomcat</artifactId>--> + <!--</exclusion>--> + <exclusion> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-logging</artifactId> + </exclusion> + <!--<exclusion>--> + <!--<groupId>ch.qos.logback</groupId>--> + <!--<artifactId>logback-classic</artifactId>--> + <!--</exclusion>--> + <!--<exclusion>--> + <!--<groupId>org.slf4j</groupId>--> + <!--<artifactId>log4j-over-slf4j</artifactId>--> + <!--</exclusion>--> + </exclusions> </dependency> + <!--<dependency>--> + <!--<groupId>org.springframework.boot</groupId>--> + <!--<artifactId>spring-boot-starter-tomcat</artifactId>--> + <!--<version>1.1.2.RELEASE</version>--> + <!--</dependency>--> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-data-jpa</artifactId> @@ -131,10 +159,34 @@ under the License. <version>${hadoop.version}</version> <!--<scope>provided</scope>--> <exclusions> - <!--<exclusion>--> - <!--<groupId>javax.servlet</groupId>--> - <!--<artifactId>servlet-api</artifactId>--> - <!--</exclusion>--> + <exclusion> + <groupId>javax.servlet</groupId> + <artifactId>servlet-api</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-common</artifactId> + <version>${hive.version}</version> + <scope>provided</scope> + <exclusions> + <exclusion> + <groupId>org.eclipse.jetty.aggregate</groupId> + <artifactId>jetty-all</artifactId> + </exclusion> + <exclusion> + <groupId>org.eclipse.jetty.orbit</groupId> + <artifactId>javax.servlet</artifactId> + </exclusion> + <exclusion> + <groupId>javax.servlet</groupId> + <artifactId>servlet-api</artifactId> + </exclusion> + <exclusion> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + </exclusion> <exclusion> <groupId>org.slf4j</groupId> <artifactId>slf4j-log4j12</artifactId> @@ -147,17 +199,58 @@ under the License. <version>${hive.version}</version> <exclusions> <exclusion> + <groupId>org.apache.hive</groupId> + <artifactId>hive-common</artifactId> + </exclusion> + <exclusion> + <artifactId>log4j-web</artifactId> + <groupId>org.apache.logging.log4j</groupId> + </exclusion> + <exclusion> + <groupId>javax.servlet</groupId> + <artifactId>javax.servlet-api</artifactId> + </exclusion> + <exclusion> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + </exclusion> + + <exclusion> <groupId>org.eclipse.jetty.aggregate</groupId> <artifactId>jetty-all</artifactId> </exclusion> - <!--<exclusion>--> - <!--<groupId>org.apache.hive</groupId>--> - <!--<artifactId>hive-shims</artifactId>--> - <!--</exclusion>--> + <exclusion> + <groupId>org.eclipse.jetty.orbit</groupId> + <artifactId>javax.servlet</artifactId> + </exclusion> <exclusion> <groupId>javax.servlet</groupId> <artifactId>servlet-api</artifactId> </exclusion> + <!--<!–<exclusion>–>--> + <!--<!–<groupId>org.apache.hive</groupId>–>--> + <!--<!–<artifactId>hive-shims</artifactId>–>--> + <!--<!–</exclusion>–>--> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-slf4j-impl</artifactId> + </exclusion> + <!--<exclusion>--> + <!--<groupId>tomcat</groupId>--> + <!--<artifactId>jasper-compiler</artifactId>--> + <!--</exclusion>--> + <!--<exclusion>--> + <!--<groupId>tomcat</groupId>--> + <!--<artifactId>jasper-runtime</artifactId>--> + <!--</exclusion>--> + <!--<exclusion>--> + <!--<groupId>javax.servlet</groupId>--> + <!--<artifactId>servlet-api</artifactId>--> + <!--</exclusion>--> </exclusions> </dependency> @@ -166,12 +259,12 @@ under the License. <groupId>io.confluent</groupId> <artifactId>kafka-schema-registry-client</artifactId> <version>${confluent.version}</version> - <exclusions> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-log4j12</artifactId> - </exclusion> - </exclusions> + <!--<exclusions>--> + <!--<exclusion>--> + <!--<groupId>org.slf4j</groupId>--> + <!--<artifactId>slf4j-log4j12</artifactId>--> + <!--</exclusion>--> + <!--</exclusions>--> </dependency> <!--schedule--> @@ -284,6 +377,31 @@ under the License. </configuration> </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-shade-plugin</artifactId> + <version>3.1.1</version> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>shade</goal> + </goals> + <configuration> + <artifactSet> + <excludes> + <!--<exclude>classworlds:classworlds</exclude>--> + <!--<exclude>junit:junit</exclude>--> + <!--<exclude>jmock:*</exclude>--> + <!--<exclude>*:xml-apis</exclude>--> + <!--<exclude>org.apache.maven:lib:tests</exclude>--> + <!--<exclude>log4j:log4j:jar:</exclude>--> + </excludes> + </artifactSet> + </configuration> + </execution> + </executions> + </plugin> </plugins> </build> </project> http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/5a7793d1/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreProxy.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreProxy.java b/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreProxy.java index f632f14..8291f82 100644 --- a/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreProxy.java +++ b/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreProxy.java @@ -29,6 +29,10 @@ import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import javax.annotation.PreDestroy; +import java.io.File; +import java.net.URI; +import java.net.URL; +import java.security.CodeSource; @Component public class HiveMetaStoreProxy { @@ -49,19 +53,101 @@ public class HiveMetaStoreProxy { @Value("${hive.hmshandler.retry.interval}") private String interval; - private HiveMetaStoreClient client = null; +// private HiveMetaStoreClient client = null; + private ThriftMetastoreClient client = null; + + private static final Logger l4j = LoggerFactory.getLogger(HiveConf.class); + + private static URL checkConfigFile(File f) { + try { + return (f.exists() && f.isFile()) ? f.toURI().toURL() : null; + } catch (Throwable e) { + if (l4j.isInfoEnabled()) { + l4j.info("Error looking for config " + f, e); + } + System.err.println("Error looking for config " + f + ": " + e.getMessage()); + return null; + } + } + + private static URL findConfigFile(ClassLoader classLoader, String name, boolean doLog) { + URL result = classLoader.getResource(name); + LOGGER.warn("result: {}", result); + if (result == null) { + String confPath = System.getenv("HIVE_CONF_DIR"); + LOGGER.warn("confPath: {}", confPath); + result = checkConfigFile(new File(confPath, name)); + LOGGER.warn("result: {}", result); + if (result == null) { + String homePath = System.getenv("HIVE_HOME"); + LOGGER.warn("homePath: {}", homePath); + String nameInConf = "conf" + File.pathSeparator + name; + LOGGER.warn("nameInConf: {}", nameInConf); + result = checkConfigFile(new File(homePath, nameInConf)); + LOGGER.warn("result: {}", result); + if (result == null) { + URI jarUri = null; + try { + java.security.ProtectionDomain domain = HiveConf.class.getProtectionDomain(); + LOGGER.warn("domain: {}", domain); + CodeSource codeSource = domain.getCodeSource(); + LOGGER.warn("codeSource: {}", codeSource); + URL location = codeSource.getLocation(); + LOGGER.warn("location: {}", location); + jarUri = location.toURI(); + +// jarUri = HiveConf.class.getProtectionDomain().getCodeSource().getLocation().toURI(); + LOGGER.warn("jarUri: {}", jarUri); + } catch (Throwable e) { + if (l4j.isInfoEnabled()) { + l4j.info("Cannot get jar URI", e); + } + System.err.println("Cannot get jar URI: " + e.getMessage()); + } + File f1 = new File(jarUri); + LOGGER.warn("f1: {}", f1); + File f2 = f1.getParentFile(); + LOGGER.warn("f2: {}", f2); + File f3 = new File(f2, nameInConf); + LOGGER.warn("f3: {}", f3); + result = checkConfigFile(f3); + } + } + } + if (doLog && l4j.isInfoEnabled()) { + l4j.info("Found configuration file " + result); + } + return result; + } + +// @Bean + public ThriftMetastoreClient initHiveMetastoreClient() { + LOGGER.warn("begin client"); + LOGGER.warn(File.pathSeparator); + LOGGER.warn(uris); + +// ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + ClassLoader classLoader = HiveConf.class.getClassLoader(); + LOGGER.warn("classLoader: {}", classLoader); + if (classLoader == null) { + classLoader = HiveConf.class.getClassLoader(); + LOGGER.warn("classLoader: {}", classLoader); + } + URL url = findConfigFile(classLoader, "hive-site.xml", true); + LOGGER.warn("url: {}", url); - @Bean - public HiveMetaStoreClient initHiveMetastoreClient() { HiveConf hiveConf = new HiveConf(); + LOGGER.warn("hive conf success"); hiveConf.set("hive.metastore.local", "false"); hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, uris); hiveConf.setIntVar(HiveConf.ConfVars.HMSHANDLERATTEMPTS, attempts); hiveConf.setVar(HiveConf.ConfVars.HMSHANDLERINTERVAL, interval); try { - client = new HiveMetaStoreClient(hiveConf); - } catch (MetaException e) { + LOGGER.warn("begin hive ms client"); + client = new ThriftMetastoreClient(hiveConf); + LOGGER.warn("hive ms client success"); + } catch (Exception e) { LOGGER.error("Failed to connect hive metastore. {}", e.getMessage()); client = null; } http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/5a7793d1/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreServiceImpl.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreServiceImpl.java b/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreServiceImpl.java index 8e265e0..674defd 100644 --- a/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreServiceImpl.java +++ b/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreServiceImpl.java @@ -48,8 +48,18 @@ public class HiveMetaStoreServiceImpl implements HiveMetaStoreService { private static final Logger LOGGER = LoggerFactory.getLogger(HiveMetaStoreService.class); +// @Autowired +// private HiveMetaStoreClient client = null; + private ThriftMetastoreClient client = null; + @Autowired - private HiveMetaStoreClient client; + private HiveMetaStoreProxy proxy; + + public void initClient() { + if (client == null) { + client = proxy.initHiveMetastoreClient(); + } + } @Value("${hive.metastore.dbname}") private String defaultDbName; @@ -64,14 +74,16 @@ public class HiveMetaStoreServiceImpl implements HiveMetaStoreService { @Override @Cacheable public Iterable<String> getAllDatabases() { + initClient(); Iterable<String> results = null; try { if (client == null) { LOGGER.warn("Hive client is null. Please check your hive config."); return new ArrayList<>(); } - results = client.getAllDatabases(); - } catch (MetaException e) { + results = client.getClient().get_all_databases(); +// results = client.getAllDatabases(); + } catch (Exception e) { reconnect(); LOGGER.error("Can not get databases : {}", e.getMessage()); } @@ -82,13 +94,15 @@ public class HiveMetaStoreServiceImpl implements HiveMetaStoreService { @Override @Cacheable public Iterable<String> getAllTableNames(String dbName) { + initClient(); Iterable<String> results = null; try { if (client == null) { LOGGER.warn("Hive client is null. Please check your hive config."); return new ArrayList<>(); } - results = client.getAllTables(getUseDbName(dbName)); + results = client.getClient().get_all_tables(dbName); +// results = client.getAllTables(getUseDbName(dbName)); } catch (Exception e) { reconnect(); LOGGER.error("Exception fetching tables info: {}", e.getMessage()); @@ -107,6 +121,7 @@ public class HiveMetaStoreServiceImpl implements HiveMetaStoreService { @Override @Cacheable public Map<String, List<Table>> getAllTable() { + initClient(); Map<String, List<Table>> results = new HashMap<>(); Iterable<String> dbs; // if hive.metastore.uris in application.properties configs wrong, client will be injected failure and will be null. @@ -128,13 +143,15 @@ public class HiveMetaStoreServiceImpl implements HiveMetaStoreService { @Override @Cacheable public Table getTable(String dbName, String tableName) { + initClient(); Table result = null; try { if (client == null) { LOGGER.warn("Hive client is null. Please check your hive config."); return null; } - result = client.getTable(getUseDbName(dbName), tableName); + result = client.getClient().get_table(getUseDbName(dbName), tableName); +// result = client.getTable(getUseDbName(dbName), tableName); } catch (Exception e) { reconnect(); LOGGER.error("Exception fetching table info : {}. {}", tableName, e.getMessage()); @@ -152,6 +169,7 @@ public class HiveMetaStoreServiceImpl implements HiveMetaStoreService { private List<Table> getTables(String db) { + initClient(); String useDbName = getUseDbName(db); List<Table> allTables = new ArrayList<>(); try { @@ -159,9 +177,11 @@ public class HiveMetaStoreServiceImpl implements HiveMetaStoreService { LOGGER.warn("Hive client is null. Please check your hive config."); return allTables; } - Iterable<String> tables = client.getAllTables(useDbName); + Iterable<String> tables = client.getClient().get_all_tables(useDbName); +// Iterable<String> tables = client.getAllTables(useDbName); for (String table : tables) { - Table tmp = client.getTable(db, table); + Table tmp = client.getClient().get_table(db, table); +// Table tmp = client.getTable(db, table); allTables.add(tmp); } } catch (Exception e) { @@ -180,12 +200,13 @@ public class HiveMetaStoreServiceImpl implements HiveMetaStoreService { } private void reconnect() { + initClient(); if (singleThreadExecutor.getActiveCount() == 0) { System.out.println("execute create thread."); singleThreadExecutor.execute(() -> { try { client.reconnect(); - } catch (MetaException e) { + } catch (Exception e) { LOGGER.error("reconnect to hive failed."); } });
