Repository: falcon Updated Branches: refs/heads/master f63a47f51 -> 833458ac1
FALCON-1017 FeedReplicationTest modified to check for _SUCCESS getting created on target directory. Contributed by Pragya M Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/444e31e3 Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/444e31e3 Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/444e31e3 Branch: refs/heads/master Commit: 444e31e37425cd25c49afaa57fc68a5152b2090d Parents: f63a47f Author: samarthg <[email protected]> Authored: Fri Feb 27 10:26:06 2015 +0530 Committer: samarthg <[email protected]> Committed: Fri Feb 27 10:26:06 2015 +0530 ---------------------------------------------------------------------- falcon-regression/CHANGES.txt | 4 ++- .../falcon/regression/core/util/HadoopUtil.java | 26 ++++++++++++++++++++ .../falcon/regression/FeedReplicationTest.java | 22 +++++++++++++++++ 3 files changed, 51 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/444e31e3/falcon-regression/CHANGES.txt ---------------------------------------------------------------------- diff --git a/falcon-regression/CHANGES.txt b/falcon-regression/CHANGES.txt index 6e6ac7f..aa33831 100644 --- a/falcon-regression/CHANGES.txt +++ b/falcon-regression/CHANGES.txt @@ -51,7 +51,9 @@ Trunk (Unreleased) via Samarth Gupta) IMPROVEMENTS - + + FALCON-1017 FeedReplicationTest modified to check for _SUCCESS getting created on + target directory(Pragya M via Samarth G) FALCON-1040 Modifying ProcessInstanceStatusTest to expose job id for running jobs in Falcon. (Pragya M via Samarth G) http://git-wip-us.apache.org/repos/asf/falcon/blob/444e31e3/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java index 64574a5..7122274 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java @@ -136,6 +136,32 @@ public final class HadoopUtil { return returnList; } + /** + * Recursively retrieves all data file names from a given location and looks for presence of availabilityFlag. + * If availabilityFlag is null then it looks for _SUCCESS file(set as default). + * @param fs filesystem + * @param location given location + * @param availabilityFlag value of availability flag set in entity + * @return + * @throws IOException + */ + public static boolean getSuccessFolder( + FileSystem fs, Path location, String availabilityFlag) throws IOException { + LOGGER.info("location : " + location); + for(FileStatus stat : fs.listStatus(location)) { + if (availabilityFlag.isEmpty()) { + if (stat.getPath().getName().equals("_SUCCESS")) { + return true; + } + } else { + if (stat.getPath().getName().equals(availabilityFlag)) { + return true; + } + } + } + return false; + } + @SuppressWarnings("deprecation") private static boolean isDir(FileStatus stat) { return stat.isDir(); http://git-wip-us.apache.org/repos/asf/falcon/blob/444e31e3/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java index b88cffc..3930c8c 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java @@ -97,6 +97,7 @@ public class FeedReplicationTest extends BaseTestClass { * Test demonstrates replication of stored data from one source cluster to one target cluster. * It checks the lifecycle of replication workflow instance including its creation. When * replication ends test checks if data was replicated correctly. + * Also checks for presence of _SUCCESS file in target directory. */ @Test public void replicate1Source1Target() @@ -164,12 +165,19 @@ public class FeedReplicationTest extends BaseTestClass { .getAllFilesRecursivelyHDFS(cluster2FS, toTarget); AssertUtil.checkForListSizes(cluster1ReplicatedData, cluster2ReplicatedData); + + //_SUCCESS does not exist in source + Assert.assertEquals(HadoopUtil.getSuccessFolder(cluster1FS, toSource, ""), false); + + //_SUCCESS should exist in target + Assert.assertEquals(HadoopUtil.getSuccessFolder(cluster2FS, toTarget, ""), true); } /** * Test demonstrates replication of stored data from one source cluster to two target clusters. * It checks the lifecycle of replication workflow instances including their creation on both * targets. When replication ends test checks if data was replicated correctly. + * Also checks for presence of _SUCCESS file in target directory. */ @Test public void replicate1Source2Targets() throws Exception { @@ -254,6 +262,13 @@ public class FeedReplicationTest extends BaseTestClass { AssertUtil.checkForListSizes(cluster1ReplicatedData, cluster2ReplicatedData); AssertUtil.checkForListSizes(cluster1ReplicatedData, cluster3ReplicatedData); + + //_SUCCESS does not exist in source + Assert.assertEquals(HadoopUtil.getSuccessFolder(cluster1FS, toSource, ""), false); + + //_SUCCESS should exist in target + Assert.assertEquals(HadoopUtil.getSuccessFolder(cluster2FS, toTarget, ""), true); + Assert.assertEquals(HadoopUtil.getSuccessFolder(cluster3FS, toTarget, ""), true); } /** @@ -262,6 +277,7 @@ public class FeedReplicationTest extends BaseTestClass { * feed still waits for availability flag (file which name is defined as availability flag in * feed definition). As soon as mentioned file is got uploaded in data directory, * replication starts and when it ends test checks if data was replicated correctly. + * Also checks for presence of availability flag in target directory. */ @Test public void availabilityFlagTest() throws Exception { @@ -349,5 +365,11 @@ public class FeedReplicationTest extends BaseTestClass { .getAllFilesRecursivelyHDFS(cluster2FS, toTarget); LOGGER.info("Data on target cluster: " + cluster2ReplicatedData); AssertUtil.checkForListSizes(cluster1ReplicatedData, cluster2ReplicatedData); + + //availabilityFlag exists in source + Assert.assertEquals(HadoopUtil.getSuccessFolder(cluster1FS, toSource, availabilityFlagName), true); + + //availabilityFlag should exist in target + Assert.assertEquals(HadoopUtil.getSuccessFolder(cluster2FS, toTarget, availabilityFlagName), true); } }
