Repository: falcon Updated Branches: refs/heads/0.10 3dc75dc85 -> 91dc204b6
FALCON-2090 HDFS Snapshot failed with UnknownHostException in HA mode â¦uling in HA Mode Author: bvellanki <[email protected]> Reviewers: "Peeyush B <[email protected]>" Closes #237 from bvellanki/FALCON-2090 (cherry picked from commit 3ed804e92d1434e42a057dc68a6faa6d2fb248ce) Signed-off-by: bvellanki <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/91dc204b Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/91dc204b Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/91dc204b Branch: refs/heads/0.10 Commit: 91dc204b60c0403d16899a44d43ce3e22e9c9960 Parents: 3dc75dc Author: bvellanki <[email protected]> Authored: Thu Jul 21 11:44:14 2016 -0700 Committer: bvellanki <[email protected]> Committed: Thu Jul 21 11:44:24 2016 -0700 ---------------------------------------------------------------------- .../replication/HdfsSnapshotReplicator.java | 8 ++++++-- .../snapshots/retention/HdfsSnapshotEvictor.java | 6 ++++-- .../falcon/snapshots/util/HdfsSnapshotUtil.java | 15 ++++++++++----- .../replication/HdfsSnapshotReplicatorTest.java | 12 ++++++------ .../java/org/apache/falcon/entity/ClusterHelper.java | 5 ++--- 5 files changed, 28 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/91dc204b/addons/hdfs-snapshot-mirroring/src/main/java/org/apache/falcon/snapshots/replication/HdfsSnapshotReplicator.java ---------------------------------------------------------------------- diff --git a/addons/hdfs-snapshot-mirroring/src/main/java/org/apache/falcon/snapshots/replication/HdfsSnapshotReplicator.java b/addons/hdfs-snapshot-mirroring/src/main/java/org/apache/falcon/snapshots/replication/HdfsSnapshotReplicator.java index 2e41cc0..6f5defe 100644 --- a/addons/hdfs-snapshot-mirroring/src/main/java/org/apache/falcon/snapshots/replication/HdfsSnapshotReplicator.java +++ b/addons/hdfs-snapshot-mirroring/src/main/java/org/apache/falcon/snapshots/replication/HdfsSnapshotReplicator.java @@ -73,8 +73,12 @@ public class HdfsSnapshotReplicator extends Configured implements Tool { String sourceStorageUrl = cmd.getOptionValue(HdfsSnapshotMirrorProperties.SOURCE_NN.getName()); String targetStorageUrl = cmd.getOptionValue(HdfsSnapshotMirrorProperties.TARGET_NN.getName()); - DistributedFileSystem sourceFs = HdfsSnapshotUtil.getSourceFileSystem(cmd); - DistributedFileSystem targetFs = HdfsSnapshotUtil.getTargetFileSystem(cmd); + // Always add to getConf() so that configuration set by oozie action is + // available when creating DistributedFileSystem. + DistributedFileSystem sourceFs = HdfsSnapshotUtil.getSourceFileSystem(cmd, + new Configuration(getConf())); + DistributedFileSystem targetFs = HdfsSnapshotUtil.getTargetFileSystem(cmd, + new Configuration(getConf())); String currentSnapshotName = HdfsSnapshotUtil.SNAPSHOT_PREFIX + cmd.getOptionValue(HdfsSnapshotMirrorProperties.SNAPSHOT_JOB_NAME.getName()) http://git-wip-us.apache.org/repos/asf/falcon/blob/91dc204b/addons/hdfs-snapshot-mirroring/src/main/java/org/apache/falcon/snapshots/retention/HdfsSnapshotEvictor.java ---------------------------------------------------------------------- diff --git a/addons/hdfs-snapshot-mirroring/src/main/java/org/apache/falcon/snapshots/retention/HdfsSnapshotEvictor.java b/addons/hdfs-snapshot-mirroring/src/main/java/org/apache/falcon/snapshots/retention/HdfsSnapshotEvictor.java index 22e3377..a50e770 100644 --- a/addons/hdfs-snapshot-mirroring/src/main/java/org/apache/falcon/snapshots/retention/HdfsSnapshotEvictor.java +++ b/addons/hdfs-snapshot-mirroring/src/main/java/org/apache/falcon/snapshots/retention/HdfsSnapshotEvictor.java @@ -60,8 +60,10 @@ public class HdfsSnapshotEvictor extends Configured implements Tool { @Override public int run(String[] args) throws Exception { CommandLine cmd = getCommand(args); - DistributedFileSystem sourceFs = HdfsSnapshotUtil.getSourceFileSystem(cmd); - DistributedFileSystem targetFs = HdfsSnapshotUtil.getTargetFileSystem(cmd); + DistributedFileSystem sourceFs = HdfsSnapshotUtil.getSourceFileSystem(cmd, + new Configuration(getConf())); + DistributedFileSystem targetFs = HdfsSnapshotUtil.getTargetFileSystem(cmd, + new Configuration(getConf())); String sourceDir = cmd.getOptionValue(HdfsSnapshotMirrorProperties.SOURCE_SNAPSHOT_DIR.getName()); String targetDir = cmd.getOptionValue(HdfsSnapshotMirrorProperties.TARGET_SNAPSHOT_DIR.getName()); http://git-wip-us.apache.org/repos/asf/falcon/blob/91dc204b/addons/hdfs-snapshot-mirroring/src/main/java/org/apache/falcon/snapshots/util/HdfsSnapshotUtil.java ---------------------------------------------------------------------- diff --git a/addons/hdfs-snapshot-mirroring/src/main/java/org/apache/falcon/snapshots/util/HdfsSnapshotUtil.java b/addons/hdfs-snapshot-mirroring/src/main/java/org/apache/falcon/snapshots/util/HdfsSnapshotUtil.java index 5196791..88f6fd9 100644 --- a/addons/hdfs-snapshot-mirroring/src/main/java/org/apache/falcon/snapshots/util/HdfsSnapshotUtil.java +++ b/addons/hdfs-snapshot-mirroring/src/main/java/org/apache/falcon/snapshots/util/HdfsSnapshotUtil.java @@ -19,6 +19,7 @@ package org.apache.falcon.snapshots.util; import org.apache.commons.cli.CommandLine; +import org.apache.commons.lang3.StringUtils; import org.apache.falcon.FalconException; import org.apache.falcon.entity.ClusterHelper; import org.apache.falcon.extensions.mirroring.hdfsSnapshot.HdfsSnapshotMirrorProperties; @@ -37,29 +38,33 @@ public final class HdfsSnapshotUtil { private HdfsSnapshotUtil() {} - public static DistributedFileSystem getSourceFileSystem(CommandLine cmd) throws FalconException { + public static DistributedFileSystem getSourceFileSystem(CommandLine cmd, + Configuration conf) throws FalconException { String sourceStorageUrl = cmd.getOptionValue(HdfsSnapshotMirrorProperties.SOURCE_NN.getName()); String sourceExecuteEndpoint = cmd.getOptionValue(HdfsSnapshotMirrorProperties.SOURCE_EXEC_URL.getName()); String sourcePrincipal = parseKerberosPrincipal(cmd.getOptionValue( HdfsSnapshotMirrorProperties.SOURCE_NN_KERBEROS_PRINCIPAL.getName())); - Configuration sourceConf = ClusterHelper.getConfiguration(sourceStorageUrl, + + Configuration sourceConf = ClusterHelper.getConfiguration(conf, sourceStorageUrl, sourceExecuteEndpoint, sourcePrincipal); return HadoopClientFactory.get().createDistributedProxiedFileSystem(sourceConf); } - public static DistributedFileSystem getTargetFileSystem(CommandLine cmd) throws FalconException { + public static DistributedFileSystem getTargetFileSystem(CommandLine cmd, + Configuration conf) throws FalconException { String targetStorageUrl = cmd.getOptionValue(HdfsSnapshotMirrorProperties.TARGET_NN.getName()); String taregtExecuteEndpoint = cmd.getOptionValue(HdfsSnapshotMirrorProperties.TARGET_EXEC_URL.getName()); String targetPrincipal = parseKerberosPrincipal(cmd.getOptionValue( HdfsSnapshotMirrorProperties.TARGET_NN_KERBEROS_PRINCIPAL.getName())); - Configuration targetConf = ClusterHelper.getConfiguration(targetStorageUrl, + Configuration targetConf = ClusterHelper.getConfiguration(conf, targetStorageUrl, taregtExecuteEndpoint, targetPrincipal); return HadoopClientFactory.get().createDistributedProxiedFileSystem(targetConf); } public static String parseKerberosPrincipal(String principal) { - if (principal.equals(HdfsSnapshotMirroringExtension.EMPTY_KERBEROS_PRINCIPAL)) { + if (StringUtils.isEmpty(principal) + || principal.equals(HdfsSnapshotMirroringExtension.EMPTY_KERBEROS_PRINCIPAL)) { return null; } return principal; http://git-wip-us.apache.org/repos/asf/falcon/blob/91dc204b/addons/hdfs-snapshot-mirroring/src/test/java/org/apache/falcon/snapshots/replication/HdfsSnapshotReplicatorTest.java ---------------------------------------------------------------------- diff --git a/addons/hdfs-snapshot-mirroring/src/test/java/org/apache/falcon/snapshots/replication/HdfsSnapshotReplicatorTest.java b/addons/hdfs-snapshot-mirroring/src/test/java/org/apache/falcon/snapshots/replication/HdfsSnapshotReplicatorTest.java index 7924214..fe7ced5 100644 --- a/addons/hdfs-snapshot-mirroring/src/test/java/org/apache/falcon/snapshots/replication/HdfsSnapshotReplicatorTest.java +++ b/addons/hdfs-snapshot-mirroring/src/test/java/org/apache/falcon/snapshots/replication/HdfsSnapshotReplicatorTest.java @@ -25,7 +25,7 @@ import org.apache.falcon.entity.store.ConfigurationStore; import org.apache.falcon.entity.v0.EntityType; import org.apache.falcon.entity.v0.cluster.Cluster; import org.apache.falcon.extensions.mirroring.hdfsSnapshot.HdfsSnapshotMirrorProperties; -import org.apache.falcon.hadoop.HadoopClientFactory; +import org.apache.falcon.snapshots.util.HdfsSnapshotUtil; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsAction; @@ -72,6 +72,7 @@ public class HdfsSnapshotReplicatorTest extends HdfsSnapshotReplicator { @BeforeClass public void init() throws Exception { + this.setConf(new Configuration()); baseDir = Files.createTempDirectory("test_snapshot-replication").toFile().getAbsoluteFile(); miniDFSCluster = MiniHdfsClusterUtil.initMiniDfs(MiniHdfsClusterUtil.SNAPSHOT_REPL_TEST_PORT, baseDir); miniDfs = miniDFSCluster.getFileSystem(); @@ -100,14 +101,13 @@ public class HdfsSnapshotReplicatorTest extends HdfsSnapshotReplicator { @Test public void replicationTest() throws Exception { - Configuration sourceConf = ClusterHelper.getConfiguration(sourceCluster); - this.setConf(sourceConf); - Configuration targetConf = ClusterHelper.getConfiguration(targetCluster); sourceStorageUrl = ClusterHelper.getStorageUrl(sourceCluster); targetStorageUrl = ClusterHelper.getStorageUrl(targetCluster); - DistributedFileSystem sourceFs = HadoopClientFactory.get().createDistributedProxiedFileSystem(sourceConf); - DistributedFileSystem targetFs = HadoopClientFactory.get().createDistributedProxiedFileSystem(targetConf); + DistributedFileSystem sourceFs = HdfsSnapshotUtil.getSourceFileSystem(cmd, + new Configuration(getConf())); + DistributedFileSystem targetFs = HdfsSnapshotUtil.getTargetFileSystem(cmd, + new Configuration(getConf())); // create dir1, create snapshot, invoke copy, check file in target, create snapshot on target Path dir1 = new Path(sourceDir, "dir1"); http://git-wip-us.apache.org/repos/asf/falcon/blob/91dc204b/common/src/main/java/org/apache/falcon/entity/ClusterHelper.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/falcon/entity/ClusterHelper.java b/common/src/main/java/org/apache/falcon/entity/ClusterHelper.java index 9e16fa4..f89def3 100644 --- a/common/src/main/java/org/apache/falcon/entity/ClusterHelper.java +++ b/common/src/main/java/org/apache/falcon/entity/ClusterHelper.java @@ -75,9 +75,8 @@ public final class ClusterHelper { return conf; } - public static Configuration getConfiguration(String storageUrl, String executeEndPoint, - String kerberosPrincipal) { - Configuration conf = new Configuration(); + public static Configuration getConfiguration(Configuration conf, String storageUrl, + String executeEndPoint, String kerberosPrincipal) { conf.set(HadoopClientFactory.FS_DEFAULT_NAME_KEY, storageUrl); conf.set(HadoopClientFactory.MR_JT_ADDRESS_KEY, executeEndPoint); conf.set(HadoopClientFactory.YARN_RM_ADDRESS_KEY, executeEndPoint);
