Author: gunther
Date: Mon Feb 23 22:55:27 2015
New Revision: 1661816
URL: http://svn.apache.org/r1661816
Log:
HIVE-9654: LLAP: initialize IO during service startup, with service
configuration (Gunther Hagleitner)
Modified:
hive/branches/llap/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
hive/branches/llap/llap-client/src/java/org/apache/hadoop/hive/llap/io/api/LlapIoProxy.java
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
Modified:
hive/branches/llap/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
URL:
http://svn.apache.org/viewvc/hive/branches/llap/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java?rev=1661816&r1=1661815&r2=1661816&view=diff
==============================================================================
---
hive/branches/llap/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
(original)
+++
hive/branches/llap/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
Mon Feb 23 22:55:27 2015
@@ -73,6 +73,7 @@ import org.apache.hadoop.hive.common.io.
import org.apache.hadoop.hive.common.io.SortPrintStream;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.llap.io.api.LlapIoProxy;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.api.Index;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
@@ -80,6 +81,7 @@ import org.apache.hadoop.hive.ql.exec.Ta
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.exec.spark.session.SparkSession;
import org.apache.hadoop.hive.ql.exec.spark.session.SparkSessionManagerImpl;
+import org.apache.hadoop.hive.llap.io.api.LlapIoProxy;
import org.apache.hadoop.hive.ql.lockmgr.zookeeper.CuratorFrameworkSingleton;
import org.apache.hadoop.hive.ql.lockmgr.zookeeper.ZooKeeperHiveLockManager;
import org.apache.hadoop.hive.ql.metadata.Hive;
@@ -411,6 +413,9 @@ public class QTestUtil {
initConf();
+ LOG.info("initializing llap");
+ LlapIoProxy.initializeLlapIo(conf);
+
// Use the current directory if it is not specified
String dataDir = conf.get("test.data.files");
if (dataDir == null) {
Modified:
hive/branches/llap/llap-client/src/java/org/apache/hadoop/hive/llap/io/api/LlapIoProxy.java
URL:
http://svn.apache.org/viewvc/hive/branches/llap/llap-client/src/java/org/apache/hadoop/hive/llap/io/api/LlapIoProxy.java?rev=1661816&r1=1661815&r2=1661816&view=diff
==============================================================================
---
hive/branches/llap/llap-client/src/java/org/apache/hadoop/hive/llap/io/api/LlapIoProxy.java
(original)
+++
hive/branches/llap/llap-client/src/java/org/apache/hadoop/hive/llap/io/api/LlapIoProxy.java
Mon Feb 23 22:55:27 2015
@@ -29,23 +29,18 @@ public class LlapIoProxy {
// Llap server depends on Hive execution, so the reverse cannot be true. We
create the I/O
// singleton once (on daemon startup); the said singleton server as the IO
interface.
- private static final Object instanceLock = new Object();
private static LlapIo io = null;
-
- // TODO: temporary interface. LLAP daemon should init this once during
- // startup then others should use get w/o configuration.
- public static LlapIo getOrCreateIo(Configuration conf) {
- if (io != null) return io;
- synchronized (instanceLock) {
- if (io != null) return io;
- initializeLlapIo(conf);
- return io;
- }
+ public static LlapIo getIo() {
+ return io;
}
public static void initializeLlapIo(Configuration conf) {
- assert io == null;
+
+ if (io != null) {
+ return; // already initialized
+ }
+
try {
io = createIoImpl(conf);
} catch (IOException e) {
@@ -53,10 +48,6 @@ public class LlapIoProxy {
}
}
- public static LlapIo getIo() {
- return io;
- }
-
private static LlapIo createIoImpl(Configuration conf) throws IOException {
try {
@SuppressWarnings("unchecked")
Modified:
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
URL:
http://svn.apache.org/viewvc/hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java?rev=1661816&r1=1661815&r2=1661816&view=diff
==============================================================================
---
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
(original)
+++
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
Mon Feb 23 22:55:27 2015
@@ -26,6 +26,7 @@ import org.apache.log4j.Logger;
import org.apache.hadoop.hive.llap.daemon.ContainerRunner;
import org.apache.hadoop.hive.llap.daemon.LlapDaemonConfiguration;
import
org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.RunContainerRequestProto;
+import org.apache.hadoop.hive.llap.io.api.LlapIoProxy;
import org.apache.hadoop.hive.llap.shufflehandler.ShuffleHandler;
public class LlapDaemon extends AbstractService implements ContainerRunner {
@@ -84,6 +85,7 @@ public class LlapDaemon extends Abstract
public void serviceInit(Configuration conf) {
server.init(conf);
containerRunner.init(conf);
+ LlapIoProxy.initializeLlapIo(conf);
}
@Override
Modified:
hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
URL:
http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java?rev=1661816&r1=1661815&r2=1661816&view=diff
==============================================================================
---
hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
(original)
+++
hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
Mon Feb 23 22:55:27 2015
@@ -83,7 +83,7 @@ public class HiveInputFormat<K extends W
/**
* A cache of InputFormat instances.
*/
- private static Map<Class, InputFormat<WritableComparable, Writable>>
inputFormats
+ private static Map<Class, InputFormat<WritableComparable, Writable>>
inputFormats
= new ConcurrentHashMap<Class, InputFormat<WritableComparable,
Writable>>();
private JobConf job;
@@ -213,8 +213,7 @@ public class HiveInputFormat<K extends W
}
LOG.info("Wrapping " + inputFormat);
@SuppressWarnings("unchecked")
- // TODO: should be LlapIoProxy.getIo eventually
- LlapIo<VectorizedRowBatch> llapIo = LlapIoProxy.getOrCreateIo(conf);
+ LlapIo<VectorizedRowBatch> llapIo = LlapIoProxy.getIo();
return castInputFormat(llapIo.getInputFormat(inputFormat));
}
