Repository: falcon Updated Branches: refs/heads/master c588628aa -> ea0492dac
FALCON-2095 HiveDR fails in NN-HA config Fix validated in a two cluster configuration with NN/RM HA setup Author: Venkat Ranganathan <[email protected]> Reviewers: "Balu Vellanki <[email protected]>" Closes #240 from vrangan/FALCON-2095 Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/ea0492da Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/ea0492da Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/ea0492da Branch: refs/heads/master Commit: ea0492dacd3d0929751060bee26b1e5a61178431 Parents: c588628 Author: Venkat Ranganathan <[email protected]> Authored: Wed Jul 27 13:22:40 2016 -0700 Committer: bvellanki <[email protected]> Committed: Wed Jul 27 13:22:40 2016 -0700 ---------------------------------------------------------------------- .../main/java/org/apache/falcon/hive/HiveDRTool.java | 6 +++--- .../org/apache/falcon/hive/mapreduce/CopyReducer.java | 3 ++- .../java/org/apache/falcon/hive/util/EventUtils.java | 6 +++--- .../java/org/apache/falcon/hive/util/FileUtils.java | 12 ++++++------ 4 files changed, 14 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/ea0492da/addons/hivedr/src/main/java/org/apache/falcon/hive/HiveDRTool.java ---------------------------------------------------------------------- diff --git a/addons/hivedr/src/main/java/org/apache/falcon/hive/HiveDRTool.java b/addons/hivedr/src/main/java/org/apache/falcon/hive/HiveDRTool.java index e45b0d8..2701e02 100644 --- a/addons/hivedr/src/main/java/org/apache/falcon/hive/HiveDRTool.java +++ b/addons/hivedr/src/main/java/org/apache/falcon/hive/HiveDRTool.java @@ -143,13 +143,13 @@ public class HiveDRTool extends Configured implements Tool { LOG.info("srcStaginPath: {}", inputOptions.getSourceStagingPath()); LOG.info("tgtStaginPath: {}", inputOptions.getTargetStagingPath()); - Configuration sourceConf = FileUtils.getConfiguration(inputOptions.getSourceWriteEP(), + Configuration sourceConf = FileUtils.getConfiguration(getConf(), inputOptions.getSourceWriteEP(), inputOptions.getSourceNNKerberosPrincipal()); sourceClusterFS = FileSystem.get(sourceConf); - Configuration targetConf = FileUtils.getConfiguration(inputOptions.getTargetWriteEP(), + Configuration targetConf = FileUtils.getConfiguration(getConf(), inputOptions.getTargetWriteEP(), inputOptions.getTargetNNKerberosPrincipal()); targetClusterFS = FileSystem.get(targetConf); - jobConf = FileUtils.getConfiguration(inputOptions.getJobClusterWriteEP(), + jobConf = FileUtils.getConfiguration(getConf(), inputOptions.getJobClusterWriteEP(), inputOptions.getJobClusterNNPrincipal()); jobFS = FileSystem.get(jobConf); http://git-wip-us.apache.org/repos/asf/falcon/blob/ea0492da/addons/hivedr/src/main/java/org/apache/falcon/hive/mapreduce/CopyReducer.java ---------------------------------------------------------------------- diff --git a/addons/hivedr/src/main/java/org/apache/falcon/hive/mapreduce/CopyReducer.java b/addons/hivedr/src/main/java/org/apache/falcon/hive/mapreduce/CopyReducer.java index f4bb31c..7c415c3 100644 --- a/addons/hivedr/src/main/java/org/apache/falcon/hive/mapreduce/CopyReducer.java +++ b/addons/hivedr/src/main/java/org/apache/falcon/hive/mapreduce/CopyReducer.java @@ -48,7 +48,8 @@ public class CopyReducer extends Reducer<Text, Text, Text, Text> { @Override protected void setup(Context context) throws IOException, InterruptedException { Configuration conf = context.getConfiguration(); - FileSystem fs= FileSystem.get(FileUtils.getConfiguration( + FileSystem fs= FileSystem.get( + FileUtils.getConfiguration(context.getConfiguration(), conf.get(HiveDRArgs.TARGET_NN.getName()), conf.get(HiveDRArgs.TARGET_NN_KERBEROS_PRINCIPAL.getName()))); hiveDRStore = new HiveDRStatusStore(fs); http://git-wip-us.apache.org/repos/asf/falcon/blob/ea0492da/addons/hivedr/src/main/java/org/apache/falcon/hive/util/EventUtils.java ---------------------------------------------------------------------- diff --git a/addons/hivedr/src/main/java/org/apache/falcon/hive/util/EventUtils.java b/addons/hivedr/src/main/java/org/apache/falcon/hive/util/EventUtils.java index 590a7e3..05b5f96 100644 --- a/addons/hivedr/src/main/java/org/apache/falcon/hive/util/EventUtils.java +++ b/addons/hivedr/src/main/java/org/apache/falcon/hive/util/EventUtils.java @@ -142,9 +142,9 @@ public class EventUtils { LOG.info("Initializing staging directory"); sourceStagingUri = new Path(sourceNN, sourceStagingPath).toString(); targetStagingUri = new Path(targetNN, targetStagingPath).toString(); - sourceFileSystem = FileSystem.get(FileUtils.getConfiguration(sourceNN, sourceNNKerberosPrincipal)); - jobFileSystem = FileSystem.get(FileUtils.getConfiguration(jobNN, jobNNKerberosPrincipal)); - targetFileSystem = FileSystem.get(FileUtils.getConfiguration(targetNN, targetNNKerberosPrincipal)); + sourceFileSystem = FileSystem.get(FileUtils.getConfiguration(conf, sourceNN, sourceNNKerberosPrincipal)); + jobFileSystem = FileSystem.get(FileUtils.getConfiguration(conf, jobNN, jobNNKerberosPrincipal)); + targetFileSystem = FileSystem.get(FileUtils.getConfiguration(conf, targetNN, targetNNKerberosPrincipal)); } private String readEvents(Path eventFileName) throws IOException { http://git-wip-us.apache.org/repos/asf/falcon/blob/ea0492da/addons/hivedr/src/main/java/org/apache/falcon/hive/util/FileUtils.java ---------------------------------------------------------------------- diff --git a/addons/hivedr/src/main/java/org/apache/falcon/hive/util/FileUtils.java b/addons/hivedr/src/main/java/org/apache/falcon/hive/util/FileUtils.java index 8b5c865..dae4849 100644 --- a/addons/hivedr/src/main/java/org/apache/falcon/hive/util/FileUtils.java +++ b/addons/hivedr/src/main/java/org/apache/falcon/hive/util/FileUtils.java @@ -42,14 +42,14 @@ public final class FileUtils { private FileUtils() {} - public static Configuration getConfiguration(final String writeEP, - final String nnKerberosPrincipal) throws IOException { - Configuration conf = HiveDRUtils.getDefaultConf(); - conf.set("fs.defaultFS", writeEP); + public static Configuration getConfiguration(Configuration conf, + final String writeEP, final String nnKerberosPrincipal) throws IOException { + Configuration newConf = new Configuration(conf); + newConf.set("fs.defaultFS", writeEP); if (StringUtils.isNotEmpty(nnKerberosPrincipal)) { - conf.set("dfs.namenode.kerberos.principal", nnKerberosPrincipal); + newConf.set("dfs.namenode.kerberos.principal", nnKerberosPrincipal); } - return conf; + return newConf; } public static void validatePath(final FileSystem fileSystem, final Path basePath) throws IOException {
