Repository: hive Updated Branches: refs/heads/master 93de95041 -> cd715e7f4
HIVE-20521 : HS2 doAs=true has permission issue with hadoop.tmp.dir, with MR and S3A filesystem (Thejas Nair, reviewed by Zoltan Haindrich) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/cd715e7f Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/cd715e7f Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/cd715e7f Branch: refs/heads/master Commit: cd715e7f4c9fc6ff1e5cbc2a2e832aef1b750424 Parents: 93de950 Author: Thejas M Nair <[email protected]> Authored: Tue Sep 11 08:45:30 2018 -0700 Committer: Thejas M Nair <[email protected]> Committed: Tue Sep 11 08:45:30 2018 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/hive/UtilsForTest.java | 19 +++++++++++++++++++ .../cli/TestEmbeddedThriftBinaryCLIService.java | 2 ++ .../org/apache/hive/jdbc/miniHS2/MiniHS2.java | 6 ++++++ .../hadoop/hive/ql/exec/mr/ExecDriver.java | 8 -------- 4 files changed, 27 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/cd715e7f/itests/hive-unit/src/test/java/org/apache/hadoop/hive/UtilsForTest.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/UtilsForTest.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/UtilsForTest.java index 6207d32..2699154 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/UtilsForTest.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/UtilsForTest.java @@ -18,6 +18,9 @@ package org.apache.hadoop.hive; +import java.util.Iterator; +import java.util.Map; + import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; @@ -38,4 +41,20 @@ public class UtilsForTest { + ";create=true"); } + /** + * Do the variable expansion by calling "set" on each variable. + * When MR jobs are run, under some circumstances they fail because + * the variable expansion fails after changes in Hadoop to prevent + * variable expansion for JobHistoryServer. So expanding them ahead + * so that variables like {test.tmp.dir} get expanded. + * @param hiveConf + */ + public static void expandHiveConfParams(HiveConf hiveConf) { + Iterator<Map.Entry<String, String>> iter = hiveConf.iterator(); + while (iter.hasNext()) { + String key = iter.next().getKey(); + hiveConf.set(key, hiveConf.get(key)); + } + } + } http://git-wip-us.apache.org/repos/asf/hive/blob/cd715e7f/itests/hive-unit/src/test/java/org/apache/hive/service/cli/TestEmbeddedThriftBinaryCLIService.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/TestEmbeddedThriftBinaryCLIService.java b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/TestEmbeddedThriftBinaryCLIService.java index 339c4ae..819838d 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/TestEmbeddedThriftBinaryCLIService.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/TestEmbeddedThriftBinaryCLIService.java @@ -18,6 +18,7 @@ package org.apache.hive.service.cli; +import org.apache.hadoop.hive.UtilsForTest; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hive.service.cli.thrift.EmbeddedThriftBinaryCLIService; import org.apache.hive.service.cli.thrift.ThriftCLIService; @@ -39,6 +40,7 @@ public class TestEmbeddedThriftBinaryCLIService extends CLIServiceTest { HiveConf conf = new HiveConf(); conf.setBoolean("datanucleus.schema.autoCreateTables", true); conf.setVar(HiveConf.ConfVars.HIVEMAPREDMODE, "nonstrict"); + UtilsForTest.expandHiveConfParams(conf); service.init(conf); client = new ThriftCLIServiceClient(service); } http://git-wip-us.apache.org/repos/asf/hive/blob/cd715e7f/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java b/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java index e4ac0a9..e5b100c 100644 --- a/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java +++ b/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java @@ -22,6 +22,7 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.util.HashMap; +import java.util.Iterator; import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -178,6 +179,11 @@ public class MiniHS2 extends AbstractHiveService { if (miniClusterType == MiniClusterType.MR && useMiniKdc) { throw new IOException("Can't create secure miniMr ... yet"); } + Iterator<Map.Entry<String, String>> iter = hiveConf.iterator(); + while (iter.hasNext()) { + String key = iter.next().getKey(); + hiveConf.set(key, hiveConf.get(key)); + } if (isHTTPTransMode) { hiveConf.setVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE, HS2_HTTP_MODE); } else { http://git-wip-us.apache.org/repos/asf/hive/blob/cd715e7f/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java index 1de782a..01dd93c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java @@ -27,9 +27,7 @@ import java.lang.management.MemoryMXBean; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.Properties; import org.apache.commons.lang.StringUtils; @@ -175,12 +173,6 @@ public class ExecDriver extends Task<MapredWork> implements Serializable, Hadoop CompilationOpContext opContext) { super.initialize(queryState, queryPlan, driverContext, opContext); - Iterator<Map.Entry<String, String>> iter = conf.iterator(); - while(iter.hasNext()) { - String key = iter.next().getKey(); - conf.set(key, conf.get(key)); - } - job = new JobConf(conf, ExecDriver.class); initializeFiles("tmpjars", getResource(conf, SessionState.ResourceType.JAR));
