Repository: falcon Updated Branches: refs/heads/master 1b8a93307 -> 8bd7a5c87
FALCON-1941 HiveDR fails with NN-HA enabled on both the source and target clusters Fix for HiveDR to work NN-HA enabled clusters Author: Venkat Ranganathan <[email protected]> Reviewers: "Ying Zheng <[email protected]>, Balu Vellanki <[email protected]>" Closes #133 from vrangan/FALCON-1941 Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/8bd7a5c8 Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/8bd7a5c8 Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/8bd7a5c8 Branch: refs/heads/master Commit: 8bd7a5c87f9c80119471b1b29a72067e6650c20e Parents: 1b8a933 Author: Venkat Ranganathan <[email protected]> Authored: Tue May 10 14:50:02 2016 -0700 Committer: bvellanki <[email protected]> Committed: Tue May 10 14:50:02 2016 -0700 ---------------------------------------------------------------------- .../org/apache/falcon/hive/util/FileUtils.java | 4 +-- .../apache/falcon/hive/util/HiveDRUtils.java | 34 +++++++++++++++++--- 2 files changed, 31 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/8bd7a5c8/addons/hivedr/src/main/java/org/apache/falcon/hive/util/FileUtils.java ---------------------------------------------------------------------- diff --git a/addons/hivedr/src/main/java/org/apache/falcon/hive/util/FileUtils.java b/addons/hivedr/src/main/java/org/apache/falcon/hive/util/FileUtils.java index 6bd6319..483a79c 100644 --- a/addons/hivedr/src/main/java/org/apache/falcon/hive/util/FileUtils.java +++ b/addons/hivedr/src/main/java/org/apache/falcon/hive/util/FileUtils.java @@ -40,8 +40,8 @@ public final class FileUtils { private FileUtils() {} - public static Configuration getConfiguration(final String writeEP, final String nnKerberosPrincipal) { - Configuration conf = new Configuration(); + public static Configuration getConfiguration(final String writeEP, final String nnKerberosPrincipal) throws IOException { + Configuration conf = HiveDRUtils.getDefaultConf(); conf.set("fs.defaultFS", writeEP); if (StringUtils.isNotEmpty(nnKerberosPrincipal)) { conf.set("dfs.namenode.kerberos.principal", nnKerberosPrincipal); http://git-wip-us.apache.org/repos/asf/falcon/blob/8bd7a5c8/addons/hivedr/src/main/java/org/apache/falcon/hive/util/HiveDRUtils.java ---------------------------------------------------------------------- diff --git a/addons/hivedr/src/main/java/org/apache/falcon/hive/util/HiveDRUtils.java b/addons/hivedr/src/main/java/org/apache/falcon/hive/util/HiveDRUtils.java index dff0803..ddcc178 100644 --- a/addons/hivedr/src/main/java/org/apache/falcon/hive/util/HiveDRUtils.java +++ b/addons/hivedr/src/main/java/org/apache/falcon/hive/util/HiveDRUtils.java @@ -25,6 +25,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.util.Shell; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; @@ -56,6 +58,8 @@ public final class HiveDRUtils { public static final String SEPARATOR = File.separator; + private static final Logger LOG = LoggerFactory.getLogger(HiveDRUtils.class); + private HiveDRUtils() {} public static ReplicationType getReplicationType(List<String> sourceTables) { @@ -65,12 +69,32 @@ public final class HiveDRUtils { public static Configuration getDefaultConf() throws IOException { Configuration conf = new Configuration(); - conf.addResource(new Path("file:///", System.getProperty("oozie.action.conf.xml"))); - String delegationToken = getFilePathFromEnv("HADOOP_TOKEN_FILE_LOCATION"); - if (delegationToken != null) { - conf.set("mapreduce.job.credentials.binary", delegationToken); - conf.set("tez.credentials.path", delegationToken); + Path confPath = new Path("file:///", System.getProperty("oozie.action.conf.xml")); + + final boolean actionConfExists = confPath.getFileSystem(conf).exists(confPath); + LOG.info("Oozie Action conf {} found ? {}", confPath, actionConfExists); + if (actionConfExists) { + LOG.info("Oozie Action conf found, adding path={}, conf={}", confPath, conf.toString()); + conf.addResource(confPath); } + + String tokenFile = System.getenv("HADOOP_TOKEN_FILE_LOCATION"); + if (StringUtils.isNotBlank(tokenFile)) { + if (Shell.WINDOWS) { + if (tokenFile.charAt(0) == '"') { + tokenFile = tokenFile.substring(1); + } + if (tokenFile.charAt(tokenFile.length() - 1) == '"') { + tokenFile = tokenFile.substring(0, tokenFile.length() - 1); + } + } + + conf.set("mapreduce.job.credentials.binary", tokenFile); + System.setProperty("mapreduce.job.credentials.binary", tokenFile); + conf.set("tez.credentials.path", tokenFile); + System.setProperty("tez.credentials.path", tokenFile); + } + return conf; }
