HIVE-11923: allow qtests to run via a single client session for tez and llap (Siddharth Seth reviewed by Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7a3e7457 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7a3e7457 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7a3e7457 Branch: refs/heads/master Commit: 7a3e74579f33ef96abcdf7f45c507c6d47b912c7 Parents: bf8acfb Author: Prasanth Jayachandran <[email protected]> Authored: Mon Sep 28 19:40:05 2015 -0500 Committer: Prasanth Jayachandran <[email protected]> Committed: Mon Sep 28 19:40:05 2015 -0500 ---------------------------------------------------------------------- data/conf/hive-log4j2.xml | 27 ++++++++++++++++++++ data/conf/llap/tez-site.xml | 6 +++++ data/conf/tez/tez-site.xml | 6 +++++ .../org/apache/hadoop/hive/ql/QTestUtil.java | 16 ++++++++++++ 4 files changed, 55 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/7a3e7457/data/conf/hive-log4j2.xml ---------------------------------------------------------------------- diff --git a/data/conf/hive-log4j2.xml b/data/conf/hive-log4j2.xml index 48a584f..860d756 100644 --- a/data/conf/hive-log4j2.xml +++ b/data/conf/hive-log4j2.xml @@ -72,6 +72,33 @@ <AppenderRef ref="EventCounter" /> </Root> + <Logger name="org.apache.hadoop.ipc" level="WARN"> + <AppenderRef ref="${sys:hive.root.logger}"/> + </Logger> + <Logger name="org.apache.hadoop.security" level="INFO"> + <AppenderRef ref="${sys:hive.root.logger}"/> + </Logger> + <Logger name="org.apache.hadoop.hdfs" level="INFO"> + <AppenderRef ref="${sys:hive.root.logger}"/> + </Logger> + <Logger name="org.apache.hadoop.hdfs.server" level="WARN"> + <AppenderRef ref="${sys:hive.root.logger}"/> + </Logger> + <Logger name="org.apache.hadoop.metrics2" level="INFO"> + <AppenderRef ref="${sys:hive.root.logger}"/> + </Logger> + <Logger name="org.mortbay" level="INFO"> + <AppenderRef ref="${sys:hive.root.logger}"/> + </Logger> + <Logger name="org.apache.hadoop.yarn" level="INFO"> + <AppenderRef ref="${sys:hive.root.logger}"/> + </Logger> + <Logger name="org.apache.hadoop.yarn.server" level="WARN"> + <AppenderRef ref="${sys:hive.root.logger}"/> + </Logger> + <Logger name="org.apache.tez" level="INFO"> + <AppenderRef ref="${sys:hive.root.logger}"/> + </Logger> <Logger name="org.apache.hadoop.conf.Configuration" level="ERROR"> <AppenderRef ref="${sys:hive.root.logger}"/> </Logger> http://git-wip-us.apache.org/repos/asf/hive/blob/7a3e7457/data/conf/llap/tez-site.xml ---------------------------------------------------------------------- diff --git a/data/conf/llap/tez-site.xml b/data/conf/llap/tez-site.xml new file mode 100644 index 0000000..940f390 --- /dev/null +++ b/data/conf/llap/tez-site.xml @@ -0,0 +1,6 @@ +<configuration> + <property> + <name>tez.am.dag.scheduler.class</name> + <value>org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrderControlled</value> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/hive/blob/7a3e7457/data/conf/tez/tez-site.xml ---------------------------------------------------------------------- diff --git a/data/conf/tez/tez-site.xml b/data/conf/tez/tez-site.xml new file mode 100644 index 0000000..940f390 --- /dev/null +++ b/data/conf/tez/tez-site.xml @@ -0,0 +1,6 @@ +<configuration> + <property> + <name>tez.am.dag.scheduler.class</name> + <value>org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrderControlled</value> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/hive/blob/7a3e7457/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index 4cab161..477a08d 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -81,6 +81,7 @@ import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.exec.spark.session.SparkSession; import org.apache.hadoop.hive.ql.exec.spark.session.SparkSessionManagerImpl; +import org.apache.hadoop.hive.ql.exec.tez.TezSessionState; import org.apache.hadoop.hive.ql.lockmgr.zookeeper.CuratorFrameworkSingleton; import org.apache.hadoop.hive.ql.lockmgr.zookeeper.ZooKeeperHiveLockManager; import org.apache.hadoop.hive.ql.metadata.Hive; @@ -158,6 +159,7 @@ public class QTestUtil { private final boolean miniMr = false; private String hadoopVer = null; private QTestSetup setup = null; + private TezSessionState tezSessionState = null; private SparkSession sparkSession = null; private boolean isSessionStateStarted = false; private static final String javaVersion = getJavaVersion(); @@ -428,10 +430,16 @@ public class QTestUtil { String uriString = WindowsPathUtil.getHdfsUriString(fs.getUri().toString()); if (clusterType == MiniClusterType.tez) { + if (confDir != null && !confDir.isEmpty()) { + conf.addResource(new URL("file://" + new File(confDir).toURI().getPath() + + "/tez-site.xml")); + } mr = shims.getMiniTezCluster(conf, 4, uriString, false); } else if (clusterType == MiniClusterType.llap) { if (confDir != null && !confDir.isEmpty()) { conf.addResource(new URL("file://" + new File(confDir).toURI().getPath() + + "/tez-site.xml")); + conf.addResource(new URL("file://" + new File(confDir).toURI().getPath() + "/llap-daemon-site.xml")); } mr = shims.getMiniTezCluster(conf, 2, uriString, true); @@ -952,6 +960,10 @@ public class QTestUtil { sparkSession = oldSs.getSparkSession(); ss.setSparkSession(sparkSession); oldSs.setSparkSession(null); + // Copy the tezSessionState from the old CliSessionState. + tezSessionState = oldSs.getTezSession(); + ss.setTezSession(tezSessionState); + oldSs.setTezSession(null); oldSs.close(); } @@ -1016,6 +1028,10 @@ public class QTestUtil { sparkSession = oldSs.getSparkSession(); ss.setSparkSession(sparkSession); oldSs.setSparkSession(null); + // Copy the tezSessionState from the old CliSessionState. + tezSessionState = oldSs.getTezSession(); + ss.setTezSession(tezSessionState); + oldSs.setTezSession(null); oldSs.close(); } if (oldSs != null && oldSs.out != null && oldSs.out != System.out) {
