Author: thejas
Date: Wed Jul 16 21:05:43 2014
New Revision: 1611190
URL: http://svn.apache.org/r1611190
Log:
HIVE-7342 : support hiveserver2,metastore specific config files (Thejas Nair,
reviewed by Jason, Sushanth, Prasad)
Modified:
hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/trunk/data/conf/hive-site.xml
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
hive/trunk/service/src/java/org/apache/hive/service/cli/thrift/EmbeddedThriftBinaryCLIService.java
hive/trunk/service/src/java/org/apache/hive/service/server/HiveServer2.java
Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL:
http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1611190&r1=1611189&r2=1611190&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
(original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Wed
Jul 16 21:05:43 2014
@@ -57,16 +57,23 @@ public class HiveConf extends Configurat
protected Properties origProp;
protected String auxJars;
private static final Log l4j = LogFactory.getLog(HiveConf.class);
+ private static boolean loadMetastoreConfig = false;
+ private static boolean loadHiveServer2Config = false;
private static URL hiveDefaultURL = null;
private static URL hiveSiteURL = null;
+ private static URL hivemetastoreSiteUrl = null;
+ private static URL hiveServer2SiteUrl = null;
+
private static byte[] confVarByteArray = null;
+
private static final Map<String, ConfVars> vars = new HashMap<String,
ConfVars>();
private final List<String> restrictList = new ArrayList<String>();
private boolean isWhiteListRestrictionEnabled = false;
private final List<String> modWhiteList = new ArrayList<String>();
+
static {
ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
if (classLoader == null) {
@@ -77,6 +84,9 @@ public class HiveConf extends Configurat
// Look for hive-site.xml on the CLASSPATH and log its location if found.
hiveSiteURL = classLoader.getResource("hive-site.xml");
+ hivemetastoreSiteUrl = classLoader.getResource("hivemetastore-site.xml");
+ hiveServer2SiteUrl = classLoader.getResource("hiveserver2-site.xml");
+
for (ConfVars confVar : ConfVars.values()) {
vars.put(confVar.varname, confVar);
}
@@ -2052,6 +2062,26 @@ public class HiveConf extends Configurat
addResource(hiveSiteURL);
}
+ // if embedded metastore is to be used as per config so far
+ // then this is considered like the metastore server case
+ String msUri = this.getVar(HiveConf.ConfVars.METASTOREURIS);
+ if(HiveConfUtil.isEmbeddedMetaStore(msUri)){
+ setLoadMetastoreConfig(true);
+ }
+
+ // load hivemetastore-site.xml if this is metastore and file exists
+ if (isLoadMetastoreConfig() && hivemetastoreSiteUrl != null) {
+ addResource(hivemetastoreSiteUrl);
+ }
+
+ // load hiveserver2-site.xml if this is hiveserver2 and file exists
+ // metastore can be embedded within hiveserver2, in such cases
+ // the conf params in hiveserver2-site.xml will override whats defined
+ // in hivemetastore-site.xml
+ if (isLoadHiveServer2Config() && hiveServer2SiteUrl != null) {
+ addResource(hiveServer2SiteUrl);
+ }
+
// Overlay the values of any system properties whose names appear in the
list of ConfVars
applySystemProperties();
@@ -2111,7 +2141,6 @@ public class HiveConf extends Configurat
setupRestrictList();
}
-
/**
* Apply system properties to this object if the property name is defined in
ConfVars
* and the value is non-null and not an empty string.
@@ -2200,6 +2229,15 @@ public class HiveConf extends Configurat
return hiveSiteURL;
}
+ public static URL getMetastoreSiteLocation() {
+ return hivemetastoreSiteUrl;
+ }
+
+ public static URL getHiveServer2SiteLocation() {
+ return hiveServer2SiteUrl;
+ }
+
+
/**
* @return the user name set in hadoop.job.ugi param or the current user
from System
* @throws IOException
@@ -2283,4 +2321,20 @@ public class HiveConf extends Configurat
restrictList.add(ConfVars.HIVE_IN_TEST.varname);
restrictList.add(ConfVars.HIVE_CONF_RESTRICTED_LIST.varname);
}
+
+ public static boolean isLoadMetastoreConfig() {
+ return loadMetastoreConfig;
+ }
+
+ public static void setLoadMetastoreConfig(boolean loadMetastoreConfig) {
+ HiveConf.loadMetastoreConfig = loadMetastoreConfig;
+ }
+
+ public static boolean isLoadHiveServer2Config() {
+ return loadHiveServer2Config;
+ }
+
+ public static void setLoadHiveServer2Config(boolean loadHiveServer2Config) {
+ HiveConf.loadHiveServer2Config = loadHiveServer2Config;
+ }
}
Modified: hive/trunk/data/conf/hive-site.xml
URL:
http://svn.apache.org/viewvc/hive/trunk/data/conf/hive-site.xml?rev=1611190&r1=1611189&r2=1611190&view=diff
==============================================================================
--- hive/trunk/data/conf/hive-site.xml (original)
+++ hive/trunk/data/conf/hive-site.xml Wed Jul 16 21:05:43 2014
@@ -221,4 +221,19 @@
<value>false</value>
</property>
+
+<property>
+ <name>hive.dummyparam.test.server.specific.config.override</name>
+ <value>from.hive-site.xml</value>
+ <description>Using dummy param to test server specific
configuration</description>
+</property>
+
+<property>
+ <name>hive.dummyparam.test.server.specific.config.hivesite</name>
+ <value>from.hive-site.xml</value>
+ <description>Using dummy param to test server specific
configuration</description>
+</property>
+
+
+
</configuration>
Modified:
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
URL:
http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java?rev=1611190&r1=1611189&r2=1611190&view=diff
==============================================================================
---
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
(original)
+++
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
Wed Jul 16 21:05:43 2014
@@ -5066,6 +5066,7 @@ public class HiveMetaStore extends Thrif
* @param args
*/
public static void main(String[] args) throws Throwable {
+ HiveConf.setLoadMetastoreConfig(true);
HiveMetastoreCli cli = new HiveMetastoreCli();
cli.parse(args);
final boolean isCliVerbose = cli.isVerbose();
Modified:
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
URL:
http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java?rev=1611190&r1=1611189&r2=1611190&view=diff
==============================================================================
---
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
(original)
+++
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
Wed Jul 16 21:05:43 2014
@@ -34,12 +34,10 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
-import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
-import java.util.Set;
import javax.security.auth.login.LoginException;
@@ -47,9 +45,9 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.common.ObjectPair;
import org.apache.hadoop.hive.common.ValidTxnList;
-import org.apache.hadoop.hive.common.ValidTxnListImpl;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.conf.HiveConfUtil;
import org.apache.hadoop.hive.metastore.api.AbortTxnRequest;
import org.apache.hadoop.hive.metastore.api.AddPartitionsRequest;
import org.apache.hadoop.hive.metastore.api.AddPartitionsResult;
@@ -68,7 +66,6 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Function;
import org.apache.hadoop.hive.metastore.api.GetOpenTxnsInfoResponse;
-import org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse;
import org.apache.hadoop.hive.metastore.api.GetPrincipalsInRoleRequest;
import org.apache.hadoop.hive.metastore.api.GetPrincipalsInRoleResponse;
import org.apache.hadoop.hive.metastore.api.GetRoleGrantsForPrincipalRequest;
@@ -112,7 +109,6 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.api.TableStatsRequest;
import org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore;
import org.apache.hadoop.hive.metastore.api.TxnAbortedException;
-import org.apache.hadoop.hive.metastore.api.TxnInfo;
import org.apache.hadoop.hive.metastore.api.TxnOpenException;
import org.apache.hadoop.hive.metastore.api.Type;
import org.apache.hadoop.hive.metastore.api.UnknownDBException;
@@ -167,7 +163,7 @@ public class HiveMetaStoreClient impleme
this.conf = conf;
String msUri = conf.getVar(HiveConf.ConfVars.METASTOREURIS);
- localMetaStore = (msUri == null) ? true : msUri.trim().isEmpty();
+ localMetaStore = HiveConfUtil.isEmbeddedMetaStore(msUri);
if (localMetaStore) {
// instantiate the metastore server handler directly instead of
connecting
// through the network
Modified:
hive/trunk/service/src/java/org/apache/hive/service/cli/thrift/EmbeddedThriftBinaryCLIService.java
URL:
http://svn.apache.org/viewvc/hive/trunk/service/src/java/org/apache/hive/service/cli/thrift/EmbeddedThriftBinaryCLIService.java?rev=1611190&r1=1611189&r2=1611190&view=diff
==============================================================================
---
hive/trunk/service/src/java/org/apache/hive/service/cli/thrift/EmbeddedThriftBinaryCLIService.java
(original)
+++
hive/trunk/service/src/java/org/apache/hive/service/cli/thrift/EmbeddedThriftBinaryCLIService.java
Wed Jul 16 21:05:43 2014
@@ -32,6 +32,7 @@ public class EmbeddedThriftBinaryCLIServ
public EmbeddedThriftBinaryCLIService() {
super(new CLIService());
isEmbedded = true;
+ HiveConf.setLoadHiveServer2Config(true);
cliService.init(new HiveConf());
cliService.start();
}
Modified:
hive/trunk/service/src/java/org/apache/hive/service/server/HiveServer2.java
URL:
http://svn.apache.org/viewvc/hive/trunk/service/src/java/org/apache/hive/service/server/HiveServer2.java?rev=1611190&r1=1611189&r2=1611190&view=diff
==============================================================================
--- hive/trunk/service/src/java/org/apache/hive/service/server/HiveServer2.java
(original)
+++ hive/trunk/service/src/java/org/apache/hive/service/server/HiveServer2.java
Wed Jul 16 21:05:43 2014
@@ -25,7 +25,6 @@ import org.apache.hadoop.hive.common.Log
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolManager;
-import org.apache.hadoop.hive.ql.exec.tez.TezSessionState;
import org.apache.hive.common.util.HiveStringUtils;
import org.apache.hive.service.CompositeService;
import org.apache.hive.service.cli.CLIService;
@@ -45,6 +44,7 @@ public class HiveServer2 extends Composi
public HiveServer2() {
super("HiveServer2");
+ HiveConf.setLoadHiveServer2Config(true);
}
@@ -142,7 +142,7 @@ public class HiveServer2 extends Composi
// before any of the other core hive classes are loaded
String initLog4jMessage = LogUtils.initHiveLog4j();
LOG.debug(initLog4jMessage);
-
+
HiveStringUtils.startupShutdownMessage(HiveServer2.class, args, LOG);
//log debug message from "oproc" after log4j initialize properly
LOG.debug(oproc.getDebugMessage().toString());
