Repository: hive Updated Branches: refs/heads/master 47e85b73d -> 068d007b8
HIVE-19716: Set spark.local.dir for a few more HoS integration tests (Sahil Takiar, reviewed by Peter Vary) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/068d007b Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/068d007b Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/068d007b Branch: refs/heads/master Commit: 068d007b8874f9eecbf30bb2739831ad9114107a Parents: 47e85b7 Author: Sahil Takiar <takiar.sa...@gmail.com> Authored: Fri May 25 11:52:18 2018 -0700 Committer: Sahil Takiar <stak...@cloudera.com> Committed: Tue May 29 12:05:38 2018 -0500 ---------------------------------------------------------------------- .../hive/ql/exec/spark/TestSparkStatistics.java | 3 +++ .../jdbc/TestJdbcWithLocalClusterSpark.java | 3 +++ ...stMultiSessionsHS2WithLocalClusterSpark.java | 3 +++ .../session/TestSparkSessionManagerImpl.java | 24 ++++++++++++-------- .../hive/spark/client/TestSparkClient.java | 4 ++++ 5 files changed, 27 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/068d007b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkStatistics.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkStatistics.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkStatistics.java index be3b501..4413161 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkStatistics.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkStatistics.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hive.ql.session.SessionState; import org.junit.Assert; import org.junit.Test; +import java.nio.file.Paths; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -47,6 +48,8 @@ public class TestSparkStatistics { conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); conf.setVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE, "spark"); conf.set("spark.master", "local-cluster[1,2,1024]"); + conf.set("spark.local.dir", Paths.get(System.getProperty("test.tmp.dir"), + "TestSparkStatistics-local-dir").toString()); SessionState.start(conf); http://git-wip-us.apache.org/repos/asf/hive/blob/068d007b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithLocalClusterSpark.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithLocalClusterSpark.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithLocalClusterSpark.java index 2a4da20..fe8a32f 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithLocalClusterSpark.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithLocalClusterSpark.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import java.nio.file.Paths; import java.sql.Connection; import java.sql.DriverManager; import java.sql.ResultSet; @@ -76,6 +77,8 @@ public class TestJdbcWithLocalClusterSpark { // Spark requires Hive to support Hadoop3 first then Spark can start // working on Hadoop3 support. Remove this after Spark supports Hadoop3. conf.set("dfs.client.datanode-restart.timeout", "30"); + conf.set("spark.local.dir", Paths.get(System.getProperty("test.tmp.dir"), + "TestJdbcWithLocalClusterSpark-local-dir").toString()); return conf; } http://git-wip-us.apache.org/repos/asf/hive/blob/068d007b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestMultiSessionsHS2WithLocalClusterSpark.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestMultiSessionsHS2WithLocalClusterSpark.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestMultiSessionsHS2WithLocalClusterSpark.java index 9f72e51..79d56f5 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestMultiSessionsHS2WithLocalClusterSpark.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestMultiSessionsHS2WithLocalClusterSpark.java @@ -18,6 +18,7 @@ package org.apache.hive.jdbc; +import java.nio.file.Paths; import java.sql.Connection; import java.sql.DriverManager; import java.sql.ResultSet; @@ -85,6 +86,8 @@ public class TestMultiSessionsHS2WithLocalClusterSpark { // Spark requires Hive to support Hadoop3 first then Spark can start // working on Hadoop3 support. Remove this after Spark supports Hadoop3. conf.set("dfs.client.datanode-restart.timeout", "30"); + conf.set("spark.local.dir", Paths.get(System.getProperty("test.tmp.dir"), + "TestMultiSessionsHS2WithLocalClusterSpark-local-dir").toString()); return conf; } http://git-wip-us.apache.org/repos/asf/hive/blob/068d007b/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/session/TestSparkSessionManagerImpl.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/session/TestSparkSessionManagerImpl.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/session/TestSparkSessionManagerImpl.java index 5924b8b..15756da 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/session/TestSparkSessionManagerImpl.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/session/TestSparkSessionManagerImpl.java @@ -24,6 +24,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.util.StringUtils; import org.junit.Test; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import java.util.Random; @@ -49,8 +50,7 @@ public class TestSparkSessionManagerImpl { /** Tests CLI scenario where we get a single session and use it multiple times. */ @Test public void testSingleSessionMultipleUse() throws Exception { - HiveConf conf = new HiveConf(); - conf.set("spark.master", "local"); + HiveConf conf = getHiveConf(); SparkSessionManager sessionManager = SparkSessionManagerImpl.getInstance(); SparkSession sparkSession1 = sessionManager.getSession(null, conf, true); @@ -76,8 +76,7 @@ public class TestSparkSessionManagerImpl { // Shutdown existing session manager sessionManagerHS2.shutdown(); - HiveConf hiveConf = new HiveConf(); - hiveConf.set("spark.master", "local"); + HiveConf hiveConf = getHiveConf(); sessionManagerHS2.setup(hiveConf); @@ -110,8 +109,7 @@ public class TestSparkSessionManagerImpl { */ @Test public void testForceConfCloning() throws Exception { - HiveConf conf = new HiveConf(); - conf.set("spark.master", "local"); + HiveConf conf = getHiveConf(); String sparkCloneConfiguration = HiveSparkClientFactory.SPARK_CLONE_CONFIGURATION; // Clear the value of sparkCloneConfiguration @@ -133,8 +131,7 @@ public class TestSparkSessionManagerImpl { @Test public void testGetHiveException() throws Exception { - HiveConf conf = new HiveConf(); - conf.set("spark.master", "local"); + HiveConf conf = getHiveConf(); SparkSessionManager ssm = SparkSessionManagerImpl.getInstance(); SparkSessionImpl ss = (SparkSessionImpl) ssm.getSession( null, conf, true); @@ -229,8 +226,7 @@ public class TestSparkSessionManagerImpl { Random random = new Random(Thread.currentThread().getId()); String threadName = Thread.currentThread().getName(); System.out.println(threadName + " started."); - HiveConf conf = new HiveConf(); - conf.set("spark.master", "local"); + HiveConf conf = getHiveConf(); SparkSession prevSession = null; SparkSession currentSession = null; @@ -256,4 +252,12 @@ public class TestSparkSessionManagerImpl { } } } + + private HiveConf getHiveConf() { + HiveConf conf = new HiveConf(); + conf.set("spark.master", "local"); + conf.set("spark.local.dir", Paths.get(System.getProperty("test.tmp.dir"), + "TestSparkSessionManagerImpl-local-dir").toString()); + return conf; + } } http://git-wip-us.apache.org/repos/asf/hive/blob/068d007b/spark-client/src/test/java/org/apache/hive/spark/client/TestSparkClient.java ---------------------------------------------------------------------- diff --git a/spark-client/src/test/java/org/apache/hive/spark/client/TestSparkClient.java b/spark-client/src/test/java/org/apache/hive/spark/client/TestSparkClient.java index fdf882b..c134625 100644 --- a/spark-client/src/test/java/org/apache/hive/spark/client/TestSparkClient.java +++ b/spark-client/src/test/java/org/apache/hive/spark/client/TestSparkClient.java @@ -37,6 +37,7 @@ import java.io.InputStream; import java.io.PrintStream; import java.io.Serializable; import java.net.URI; +import java.nio.file.Paths; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -86,6 +87,9 @@ public class TestSparkClient { conf.put("spark.home", System.getProperty("spark.home")); } + conf.put("spark.local.dir", Paths.get(System.getProperty("test.tmp.dir"), + "TestSparkClient-local-dir").toString()); + return conf; }