Repository: incubator-falcon Updated Branches: refs/heads/master 9f1efe096 -> 4f2b524d9
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4f2b524d/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedReplicationPartitionExpTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedReplicationPartitionExpTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedReplicationPartitionExpTest.java index bde4e65..d7df953 100755 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedReplicationPartitionExpTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedReplicationPartitionExpTest.java @@ -18,6 +18,7 @@ package org.apache.falcon.regression.prism; +import org.apache.falcon.regression.Entities.FeedMerlin; import org.apache.falcon.regression.core.bundle.Bundle; import org.apache.falcon.entity.v0.EntityType; import org.apache.falcon.entity.v0.feed.ActionType; @@ -31,7 +32,6 @@ import org.apache.falcon.regression.core.util.InstanceUtil; import org.apache.falcon.regression.core.util.OSUtil; import org.apache.falcon.regression.core.util.TimeUtil; import org.apache.falcon.regression.core.util.Util; -import org.apache.falcon.regression.core.util.XmlUtil; import org.apache.falcon.regression.testHelper.BaseTestClass; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -70,10 +70,10 @@ public class PrismFeedReplicationPartitionExpTest extends BaseTestClass { private String testBaseDir2 = baseTestDir + "/clusterPath/localDC/rc/billing"; private String testBaseDir3 = baseTestDir + "/dataBillingRC/fetlrc/billing"; private String testBaseDir4 = baseTestDir + "/sourcetarget"; - private String testBaseDirServer1source = baseTestDir + "/source1"; + private String testBaseDirServer1Source = baseTestDir + "/source1"; private String testDirWithDate = testBaseDir1 + testDate; - private String testDirWithDateSourcetarget = testBaseDir4 + testDate; - private String testDirWithDateSource1 = testBaseDirServer1source + testDate; + private String testDirWithDateSourceTarget = testBaseDir4 + testDate; + private String testDirWithDateSource1 = testBaseDirServer1Source + testDate; private String testFile1 = OSUtil.RESOURCES + OSUtil.getPath("ReplicationResources", "feed-s4Replication.xml"); private String testFile2 = OSUtil.RESOURCES + OSUtil.getPath("ReplicationResources", "id.pig"); @@ -145,17 +145,17 @@ public class PrismFeedReplicationPartitionExpTest extends BaseTestClass { //data for test normalTest_1s2t_pst where both source target partition are required - uploadDataToServer3(testDirWithDateSourcetarget + "00/ua3/ua2/", testFile1); - uploadDataToServer3(testDirWithDateSourcetarget + "05/ua3/ua2/", testFile2); - uploadDataToServer3(testDirWithDateSourcetarget + "10/ua3/ua2/", testFile3); - uploadDataToServer3(testDirWithDateSourcetarget + "15/ua3/ua2/", testFile4); - uploadDataToServer3(testDirWithDateSourcetarget + "20/ua3/ua2/", testFile4); + uploadDataToServer3(testDirWithDateSourceTarget + "00/ua3/ua2/", testFile1); + uploadDataToServer3(testDirWithDateSourceTarget + "05/ua3/ua2/", testFile2); + uploadDataToServer3(testDirWithDateSourceTarget + "10/ua3/ua2/", testFile3); + uploadDataToServer3(testDirWithDateSourceTarget + "15/ua3/ua2/", testFile4); + uploadDataToServer3(testDirWithDateSourceTarget + "20/ua3/ua2/", testFile4); - uploadDataToServer3(testDirWithDateSourcetarget + "00/ua3/ua1/", testFile1); - uploadDataToServer3(testDirWithDateSourcetarget + "05/ua3/ua1/", testFile2); - uploadDataToServer3(testDirWithDateSourcetarget + "10/ua3/ua1/", testFile3); - uploadDataToServer3(testDirWithDateSourcetarget + "15/ua3/ua1/", testFile4); - uploadDataToServer3(testDirWithDateSourcetarget + "20/ua3/ua1/", testFile4); + uploadDataToServer3(testDirWithDateSourceTarget + "00/ua3/ua1/", testFile1); + uploadDataToServer3(testDirWithDateSourceTarget + "05/ua3/ua1/", testFile2); + uploadDataToServer3(testDirWithDateSourceTarget + "10/ua3/ua1/", testFile3); + uploadDataToServer3(testDirWithDateSourceTarget + "15/ua3/ua1/", testFile4); + uploadDataToServer3(testDirWithDateSourceTarget + "20/ua3/ua1/", testFile4); // data when server 1 acts as source uploadDataToServer1(testDirWithDateSource1 + "00/ua2/", testFile1); @@ -208,27 +208,33 @@ public class PrismFeedReplicationPartitionExpTest extends BaseTestClass { String feed = bundles[0].getDataSets().get(0); - feed = InstanceUtil.setFeedCluster(feed, - XmlUtil.createValidity("2012-10-01T12:00Z", "2010-01-01T00:00Z"), - XmlUtil.createRetention("days(1000000)", ActionType.DELETE), null, - ClusterType.SOURCE, null); - - feed = InstanceUtil - .setFeedCluster(feed, XmlUtil.createValidity(startTimeUA1, "2012-10-01T12:10Z"), - XmlUtil.createRetention("days(1000000)", ActionType.DELETE), - Util.readEntityName(bundles[0].getClusters().get(0)), ClusterType.SOURCE, "", - testBaseDir1 + MINUTE_DATE_PATTERN); - - feed = InstanceUtil - .setFeedCluster(feed, XmlUtil.createValidity(startTimeUA2, "2012-10-01T12:25Z"), - XmlUtil.createRetention("days(1000000)", ActionType.DELETE), - Util.readEntityName(bundles[1].getClusters().get(0)), ClusterType.TARGET, "", - testBaseDir2 + MINUTE_DATE_PATTERN); - - feed = InstanceUtil.setFeedCluster(feed, - XmlUtil.createValidity("2012-10-01T12:00Z", "2099-01-01T00:00Z"), - XmlUtil.createRetention("days(1000000)", ActionType.DELETE), - Util.readEntityName(bundles[2].getClusters().get(0)), ClusterType.SOURCE, ""); + feed = FeedMerlin.fromString(feed).clearFeedClusters().toString(); + + feed = FeedMerlin.fromString(feed).addFeedCluster( + new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[0].getClusters().get(0))) + .withRetention("days(1000000)", ActionType.DELETE) + .withValidity(startTimeUA1, "2012-10-01T12:10Z") + .withClusterType(ClusterType.SOURCE) + .withPartition("") + .withDataLocation(testBaseDir1 + MINUTE_DATE_PATTERN) + .build()).toString(); + + feed = FeedMerlin.fromString(feed).addFeedCluster( + new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[1].getClusters().get(0))) + .withRetention("days(1000000)", ActionType.DELETE) + .withValidity(startTimeUA2, "2012-10-01T12:25Z") + .withClusterType(ClusterType.TARGET) + .withPartition("") + .withDataLocation(testBaseDir2 + MINUTE_DATE_PATTERN) + .build()).toString(); + + feed = FeedMerlin.fromString(feed).addFeedCluster( + new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[2].getClusters().get(0))) + .withRetention("days(1000000)", ActionType.DELETE) + .withValidity("2012-10-01T12:00Z", "2099-01-01T00:00Z") + .withClusterType(ClusterType.SOURCE) + .withPartition("") + .build()).toString(); LOGGER.info("feed: " + Util.prettyPrintXml(feed)); @@ -240,7 +246,7 @@ public class PrismFeedReplicationPartitionExpTest extends BaseTestClass { @Test(enabled = true) - public void normalTest1s1t1nPS() throws Exception { + public void normalTest1Source1Target1NeutralPartitionedSource() throws Exception { //this test is for ideal condition when data is present in all the required places and // replication takes // place normally @@ -256,27 +262,33 @@ public class PrismFeedReplicationPartitionExpTest extends BaseTestClass { String feed = bundles[0].getDataSets().get(0); - feed = InstanceUtil.setFeedCluster(feed, - XmlUtil.createValidity("2012-10-01T12:00Z", "2010-01-01T00:00Z"), - XmlUtil.createRetention("days(100000)", ActionType.DELETE), null, - ClusterType.SOURCE, null); - - feed = InstanceUtil - .setFeedCluster(feed, XmlUtil.createValidity(startTimeUA1, "2099-10-01T12:10Z"), - XmlUtil.createRetention("days(100000)", ActionType.DELETE), - Util.readEntityName(bundles[0].getClusters().get(0)), null, null); - - feed = InstanceUtil - .setFeedCluster(feed, XmlUtil.createValidity(startTimeUA2, "2099-10-01T12:25Z"), - XmlUtil.createRetention("days(100000)", ActionType.DELETE), - Util.readEntityName(bundles[1].getClusters().get(0)), ClusterType.TARGET, null, - testBaseDir2 + MINUTE_DATE_PATTERN); - - feed = InstanceUtil - .setFeedCluster(feed, XmlUtil.createValidity("2012-10-01T12:00Z", "2099-01-01T00:00Z"), - XmlUtil.createRetention("days(100000)", ActionType.DELETE), - Util.readEntityName(bundles[2].getClusters().get(0)), ClusterType.SOURCE, - "${cluster.colo}", testBaseDir1 + MINUTE_DATE_PATTERN); + feed = FeedMerlin.fromString(feed).clearFeedClusters().toString(); + + feed = FeedMerlin.fromString(feed).addFeedCluster( + new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[0].getClusters().get(0))) + .withRetention("days(100000)", ActionType.DELETE) + .withValidity(startTimeUA1, "2099-10-01T12:10Z") + .build()) + .toString(); + + feed = FeedMerlin.fromString(feed).addFeedCluster( + new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[1].getClusters().get(0))) + .withRetention("days(100000)", ActionType.DELETE) + .withValidity(startTimeUA2, "2099-10-01T12:25Z") + .withClusterType(ClusterType.TARGET) + .withDataLocation(testBaseDir2 + MINUTE_DATE_PATTERN) + .build()) + .toString(); + + feed = FeedMerlin.fromString(feed).addFeedCluster( + new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[2].getClusters().get(0))) + .withRetention("days(100000)", ActionType.DELETE) + .withValidity("2012-10-01T12:00Z", "2099-01-01T00:00Z") + .withClusterType(ClusterType.SOURCE) + .withPartition("${cluster.colo}") + .withDataLocation(testBaseDir1 + MINUTE_DATE_PATTERN) + .build()) + .toString(); LOGGER.info("feed: " + Util.prettyPrintXml(feed)); @@ -339,7 +351,7 @@ public class PrismFeedReplicationPartitionExpTest extends BaseTestClass { @Test(enabled = true) - public void normalTest1s1t1nPT() throws Exception { + public void normalTest1Source1Target1NeutralPartitionedTarget() throws Exception { //this test is for ideal condition when data is present in all the required places and // replication takes // place normally @@ -351,27 +363,31 @@ public class PrismFeedReplicationPartitionExpTest extends BaseTestClass { String feed = bundles[0].getDataSets().get(0); - feed = InstanceUtil.setFeedCluster(feed, - XmlUtil.createValidity("2012-10-01T12:00Z", "2010-01-01T00:00Z"), - XmlUtil.createRetention("days(1000000)", ActionType.DELETE), null, - ClusterType.SOURCE, null); - - feed = InstanceUtil - .setFeedCluster(feed, XmlUtil.createValidity(startTimeUA1, "2099-10-01T12:10Z"), - XmlUtil.createRetention("days(1000000)", ActionType.DELETE), - Util.readEntityName(bundles[0].getClusters().get(0)), null, null); - - feed = InstanceUtil - .setFeedCluster(feed, XmlUtil.createValidity(startTimeUA2, "2099-10-01T12:25Z"), - XmlUtil.createRetention("days(1000000)", ActionType.DELETE), - Util.readEntityName(bundles[1].getClusters().get(0)), ClusterType.TARGET, - "${cluster.colo}", testBaseDir2 + MINUTE_DATE_PATTERN); - - feed = InstanceUtil.setFeedCluster(feed, - XmlUtil.createValidity("2012-10-01T12:00Z", "2099-01-01T00:00Z"), - XmlUtil.createRetention("days(1000000)", ActionType.DELETE), - Util.readEntityName(bundles[2].getClusters().get(0)), ClusterType.SOURCE, null, - testBaseDir1 + MINUTE_DATE_PATTERN); + feed = FeedMerlin.fromString(feed).clearFeedClusters().toString(); + + feed = FeedMerlin.fromString(feed).addFeedCluster( + new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[0].getClusters().get(0))) + .withRetention("days(1000000)", ActionType.DELETE) + .withValidity(startTimeUA1, "2099-10-01T12:10Z") + .build()).toString(); + + feed = FeedMerlin.fromString(feed).addFeedCluster( + new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[1].getClusters().get(0))) + .withRetention("days(1000000)", ActionType.DELETE) + .withValidity(startTimeUA2, "2099-10-01T12:25Z") + .withClusterType(ClusterType.TARGET) + .withPartition("${cluster.colo}") + .withDataLocation(testBaseDir2 + MINUTE_DATE_PATTERN) + .build()) + .toString(); + + feed = FeedMerlin.fromString(feed).addFeedCluster( + new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[2].getClusters().get(0))) + .withRetention("days(1000000)", ActionType.DELETE) + .withValidity("2012-10-01T12:00Z", "2099-01-01T00:00Z") + .withClusterType(ClusterType.SOURCE) + .withDataLocation(testBaseDir1 + MINUTE_DATE_PATTERN) + .build()).toString(); LOGGER.info("feed: " + Util.prettyPrintXml(feed)); @@ -423,7 +439,7 @@ public class PrismFeedReplicationPartitionExpTest extends BaseTestClass { @Test(enabled = true) - public void normalTest1s2tPT() throws Exception { + public void normalTest1Source2TargetPartitionedTarget() throws Exception { //this test is for ideal condition when data is present in all the required places and // replication takes // place normally @@ -442,27 +458,30 @@ public class PrismFeedReplicationPartitionExpTest extends BaseTestClass { String feed = bundles[0].getDataSets().get(0); feed = InstanceUtil.setFeedFilePath(feed, testBaseDir3 + MINUTE_DATE_PATTERN); - feed = InstanceUtil.setFeedCluster(feed, - XmlUtil.createValidity("2012-10-01T12:00Z", "2010-01-01T00:00Z"), - XmlUtil.createRetention("days(1000000)", ActionType.DELETE), null, - ClusterType.SOURCE, null); + feed = FeedMerlin.fromString(feed).clearFeedClusters().toString(); - feed = InstanceUtil - .setFeedCluster(feed, XmlUtil.createValidity(startTimeUA1, "2012-10-01T12:10Z"), - XmlUtil.createRetention("days(1000000)", ActionType.DELETE), - Util.readEntityName(bundles[0].getClusters().get(0)), ClusterType.TARGET, - "${cluster.colo}"); + feed = FeedMerlin.fromString(feed).addFeedCluster( + new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[0].getClusters().get(0))) + .withRetention("days(1000000)", ActionType.DELETE) + .withValidity(startTimeUA1, "2012-10-01T12:10Z") + .withClusterType(ClusterType.TARGET) + .withPartition("${cluster.colo}") + .build()).toString(); - feed = InstanceUtil - .setFeedCluster(feed, XmlUtil.createValidity(startTimeUA2, "2012-10-01T12:25Z"), - XmlUtil.createRetention("days(1000000)", ActionType.DELETE), - Util.readEntityName(bundles[1].getClusters().get(0)), ClusterType.TARGET, - "${cluster.colo}"); + feed = FeedMerlin.fromString(feed).addFeedCluster( + new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[1].getClusters().get(0))) + .withRetention("days(1000000)", ActionType.DELETE) + .withValidity(startTimeUA2, "2012-10-01T12:25Z") + .withClusterType(ClusterType.TARGET) + .withPartition("${cluster.colo}") + .build()).toString(); - feed = InstanceUtil.setFeedCluster(feed, - XmlUtil.createValidity("2012-10-01T12:00Z", "2099-01-01T00:00Z"), - XmlUtil.createRetention("days(1000000)", ActionType.DELETE), - Util.readEntityName(bundles[2].getClusters().get(0)), ClusterType.SOURCE, null); + feed = FeedMerlin.fromString(feed).addFeedCluster( + new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[2].getClusters().get(0))) + .withRetention("days(1000000)", ActionType.DELETE) + .withValidity("2012-10-01T12:00Z", "2099-01-01T00:00Z") + .withClusterType(ClusterType.SOURCE) + .build()).toString(); LOGGER.info("feed: " + Util.prettyPrintXml(feed)); @@ -521,7 +540,7 @@ public class PrismFeedReplicationPartitionExpTest extends BaseTestClass { } @Test(enabled = true, groups = "embedded") - public void normalTest2s1tPT() throws Exception { + public void normalTest2Source1TargetPartitionedTarget() throws Exception { //this test is for ideal condition when data is present in all the required places and // replication takes // place normally @@ -538,28 +557,32 @@ public class PrismFeedReplicationPartitionExpTest extends BaseTestClass { String feed = bundles[0].getDataSets().get(0); - feed = InstanceUtil.setFeedCluster(feed, - XmlUtil.createValidity("2012-10-01T12:00Z", "2010-01-01T00:00Z"), - XmlUtil.createRetention("days(1000000)", ActionType.DELETE), null, - ClusterType.SOURCE, null); - - feed = InstanceUtil - .setFeedCluster(feed, XmlUtil.createValidity(startTimeUA1, "2012-10-01T12:10Z"), - XmlUtil.createRetention("days(1000000)", ActionType.DELETE), - Util.readEntityName(bundles[0].getClusters().get(0)), ClusterType.SOURCE, null, - testBaseDir1 + MINUTE_DATE_PATTERN); - - feed = InstanceUtil - .setFeedCluster(feed, XmlUtil.createValidity(startTimeUA2, "2012-10-01T12:25Z"), - XmlUtil.createRetention("days(1000000)", ActionType.DELETE), - Util.readEntityName(bundles[1].getClusters().get(0)), ClusterType.TARGET, - "${cluster.colo}", - testBaseDir2 + MINUTE_DATE_PATTERN); - - feed = InstanceUtil.setFeedCluster(feed, - XmlUtil.createValidity("2012-10-01T12:00Z", "2099-01-01T00:00Z"), - XmlUtil.createRetention("days(1000000)", ActionType.DELETE), - Util.readEntityName(bundles[2].getClusters().get(0)), ClusterType.SOURCE, null); + feed = FeedMerlin.fromString(feed).clearFeedClusters().toString(); + + feed = FeedMerlin.fromString(feed).addFeedCluster( + new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[0].getClusters().get(0))) + .withRetention("days(1000000)", ActionType.DELETE) + .withValidity(startTimeUA1, "2012-10-01T12:10Z") + .withClusterType(ClusterType.SOURCE) + .withDataLocation(testBaseDir1 + MINUTE_DATE_PATTERN) + .build()).toString(); + + feed = FeedMerlin.fromString(feed).addFeedCluster( + new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[1].getClusters().get(0))) + .withRetention("days(1000000)", ActionType.DELETE) + .withValidity(startTimeUA2, "2012-10-01T12:25Z") + .withClusterType(ClusterType.TARGET) + .withPartition("${cluster.colo}") + .withDataLocation(testBaseDir2 + MINUTE_DATE_PATTERN) + .build()) + .toString(); + + feed = FeedMerlin.fromString(feed).addFeedCluster( + new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[2].getClusters().get(0))) + .withRetention("days(1000000)", ActionType.DELETE) + .withValidity("2012-10-01T12:00Z", "2099-01-01T00:00Z") + .withClusterType(ClusterType.SOURCE) + .build()).toString(); //clean target if old data exists LOGGER.info("feed: " + Util.prettyPrintXml(feed)); @@ -575,7 +598,7 @@ public class PrismFeedReplicationPartitionExpTest extends BaseTestClass { @Test(enabled = true) - public void normalTest1s2tPS() throws Exception { + public void normalTest1Source2TargetPartitionedSource() throws Exception { //this test is for ideal condition when data is present in all the required places and // replication takes @@ -595,28 +618,33 @@ public class PrismFeedReplicationPartitionExpTest extends BaseTestClass { String feed = bundles[0].getDataSets().get(0); feed = InstanceUtil.setFeedFilePath(feed, testBaseDir1 + MINUTE_DATE_PATTERN); - feed = InstanceUtil.setFeedCluster(feed, - XmlUtil.createValidity("2012-10-01T12:00Z", "2010-01-01T00:00Z"), - XmlUtil.createRetention("days(10000000)", ActionType.DELETE), null, - ClusterType.SOURCE, null); - - feed = InstanceUtil - .setFeedCluster(feed, XmlUtil.createValidity(startTimeUA1, "2012-10-01T12:11Z"), - XmlUtil.createRetention("days(10000000)", ActionType.DELETE), - Util.readEntityName(bundles[0].getClusters().get(0)), ClusterType.TARGET, null, - testBaseDir1 + "/ua1" + MINUTE_DATE_PATTERN); - - feed = InstanceUtil - .setFeedCluster(feed, XmlUtil.createValidity(startTimeUA2, "2012-10-01T12:26Z"), - XmlUtil.createRetention("days(10000000)", ActionType.DELETE), - Util.readEntityName(bundles[1].getClusters().get(0)), ClusterType.TARGET, null, - testBaseDir1 + "/ua2" + MINUTE_DATE_PATTERN); - - feed = InstanceUtil.setFeedCluster(feed, - XmlUtil.createValidity("2012-10-01T12:00Z", "2099-01-01T00:00Z"), - XmlUtil.createRetention("days(10000000)", ActionType.DELETE), - Util.readEntityName(bundles[2].getClusters().get(0)), ClusterType.SOURCE, - "${cluster.colo}"); + feed = FeedMerlin.fromString(feed).clearFeedClusters().toString(); + + feed = FeedMerlin.fromString(feed).addFeedCluster( + new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[0].getClusters().get(0))) + .withRetention("days(10000000)", ActionType.DELETE) + .withValidity(startTimeUA1, "2012-10-01T12:11Z") + .withClusterType(ClusterType.TARGET) + .withDataLocation(testBaseDir1 + "/ua1" + MINUTE_DATE_PATTERN) + .build()) + .toString(); + + feed = FeedMerlin.fromString(feed).addFeedCluster( + new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[1].getClusters().get(0))) + .withRetention("days(10000000)", ActionType.DELETE) + .withValidity(startTimeUA2, "2012-10-01T12:26Z") + .withClusterType(ClusterType.TARGET) + .withDataLocation(testBaseDir1 + "/ua2" + MINUTE_DATE_PATTERN) + .build()) + .toString(); + + feed = FeedMerlin.fromString(feed).addFeedCluster( + new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[2].getClusters().get(0))) + .withRetention("days(10000000)", ActionType.DELETE) + .withValidity("2012-10-01T12:00Z", "2099-01-01T00:00Z") + .withClusterType(ClusterType.SOURCE) + .withPartition("${cluster.colo}") + .build()).toString(); LOGGER.info("feed: " + Util.prettyPrintXml(feed)); @@ -678,7 +706,7 @@ public class PrismFeedReplicationPartitionExpTest extends BaseTestClass { @Test(enabled = true) - public void normalTest2s1tPS() throws Exception { + public void normalTest2Source1TargetPartitionedSource() throws Exception { //this test is for ideal condition when data is present in all the required places and // replication takes // place normally @@ -695,29 +723,34 @@ public class PrismFeedReplicationPartitionExpTest extends BaseTestClass { String startTimeUA2 = "2012-10-01T12:00Z"; String feed = bundles[0].getDataSets().get(0); - feed = InstanceUtil.setFeedCluster(feed, - XmlUtil.createValidity("2012-10-01T12:00Z", "2010-01-01T00:00Z"), - XmlUtil.createRetention("days(1000000)", ActionType.DELETE), null, - ClusterType.SOURCE, null); - - feed = InstanceUtil - .setFeedCluster(feed, XmlUtil.createValidity(startTimeUA1, "2099-10-01T12:10Z"), - XmlUtil.createRetention("days(1000000)", ActionType.DELETE), - Util.readEntityName(bundles[0].getClusters().get(0)), ClusterType.SOURCE, - "${cluster.colo}", - testBaseDirServer1source + MINUTE_DATE_PATTERN); - - feed = InstanceUtil - .setFeedCluster(feed, XmlUtil.createValidity(startTimeUA2, "2099-10-01T12:25Z"), - XmlUtil.createRetention("days(1000000)", ActionType.DELETE), - Util.readEntityName(bundles[1].getClusters().get(0)), ClusterType.TARGET, null, - testBaseDir2 + "/replicated" + MINUTE_DATE_PATTERN); - - feed = InstanceUtil.setFeedCluster(feed, - XmlUtil.createValidity("2012-10-01T12:00Z", "2099-01-01T00:00Z"), - XmlUtil.createRetention("days(1000000)", ActionType.DELETE), - Util.readEntityName(bundles[2].getClusters().get(0)), ClusterType.SOURCE, - "${cluster.colo}", testBaseDir1 + MINUTE_DATE_PATTERN); + feed = FeedMerlin.fromString(feed).clearFeedClusters().toString(); + + feed = FeedMerlin.fromString(feed).addFeedCluster( + new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[0].getClusters().get(0))) + .withRetention("days(1000000)", ActionType.DELETE) + .withValidity(startTimeUA1, "2099-10-01T12:10Z") + .withClusterType(ClusterType.SOURCE) + .withPartition("${cluster.colo}") + .withDataLocation(testBaseDirServer1Source + MINUTE_DATE_PATTERN) + .build()).toString(); + + feed = FeedMerlin.fromString(feed).addFeedCluster( + new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[1].getClusters().get(0))) + .withRetention("days(1000000)", ActionType.DELETE) + .withValidity(startTimeUA2, "2099-10-01T12:25Z") + .withClusterType(ClusterType.TARGET) + .withDataLocation(testBaseDir2 + "/replicated" + MINUTE_DATE_PATTERN) + .build()).toString(); + + feed = FeedMerlin.fromString(feed).addFeedCluster( + new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[2].getClusters().get(0))) + .withRetention("days(1000000)", ActionType.DELETE) + .withValidity("2012-10-01T12:00Z", "2099-01-01T00:00Z") + .withClusterType(ClusterType.SOURCE) + .withPartition("${cluster.colo}") + .withDataLocation(testBaseDir1 + MINUTE_DATE_PATTERN) + .build()) + .toString(); LOGGER.info("feed: " + Util.prettyPrintXml(feed)); @@ -750,7 +783,7 @@ public class PrismFeedReplicationPartitionExpTest extends BaseTestClass { List<Path> ua1OriginalData00 = HadoopUtil .getAllFilesRecursivelyHDFS(cluster1FS, new Path( - testBaseDirServer1source + testDate + "00/ua1")); + testBaseDirServer1Source + testDate + "00/ua1")); List<Path> ua3OriginalData05 = HadoopUtil .getAllFilesRecursivelyHDFS(cluster3FS, new Path(testDirWithDate + "05/ua1")); @@ -760,7 +793,7 @@ public class PrismFeedReplicationPartitionExpTest extends BaseTestClass { @Test(enabled = true) - public void normalTest1s2tPST() throws Exception { + public void normalTest1Source2TargetPartitionedSourceTarget() throws Exception { //this test is for ideal condition when data is present in all the required places and @@ -779,28 +812,35 @@ public class PrismFeedReplicationPartitionExpTest extends BaseTestClass { String feed = bundles[0].getDataSets().get(0); feed = InstanceUtil.setFeedFilePath(feed, testBaseDir1 + MINUTE_DATE_PATTERN); - feed = InstanceUtil.setFeedCluster(feed, - XmlUtil.createValidity("2012-10-01T12:00Z", "2010-01-01T00:00Z"), - XmlUtil.createRetention("days(1000000)", ActionType.DELETE), null, - ClusterType.SOURCE, null); - - feed = InstanceUtil - .setFeedCluster(feed, XmlUtil.createValidity(startTimeUA1, "2099-10-01T12:10Z"), - XmlUtil.createRetention("days(1000000)", ActionType.DELETE), - Util.readEntityName(bundles[0].getClusters().get(0)), ClusterType.TARGET, - "${cluster.colo}", testBaseDir1 + "/ua1" + MINUTE_DATE_PATTERN + "/"); - - feed = InstanceUtil - .setFeedCluster(feed, XmlUtil.createValidity(startTimeUA2, "2099-10-01T12:25Z"), - XmlUtil.createRetention("days(1000000)", ActionType.DELETE), - Util.readEntityName(bundles[1].getClusters().get(0)), ClusterType.TARGET, - "${cluster.colo}", testBaseDir1 + "/ua2" + MINUTE_DATE_PATTERN + "/"); - - feed = InstanceUtil.setFeedCluster(feed, - XmlUtil.createValidity("2012-10-01T12:00Z", "2099-01-01T00:00Z") - , XmlUtil.createRetention("days(1000000)", ActionType.DELETE), - Util.readEntityName(bundles[2].getClusters().get(0)), ClusterType.SOURCE, - "${cluster.colo}", testBaseDir4 + MINUTE_DATE_PATTERN + "/"); + feed = FeedMerlin.fromString(feed).clearFeedClusters().toString(); + + feed = FeedMerlin.fromString(feed).addFeedCluster( + new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[0].getClusters().get(0))) + .withRetention("days(1000000)", ActionType.DELETE) + .withValidity(startTimeUA1, "2099-10-01T12:10Z") + .withClusterType(ClusterType.TARGET) + .withPartition("${cluster.colo}") + .withDataLocation(testBaseDir1 + "/ua1" + MINUTE_DATE_PATTERN + "/") + .build()).toString(); + + feed = FeedMerlin.fromString(feed).addFeedCluster( + new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[1].getClusters().get(0))) + .withRetention("days(1000000)", ActionType.DELETE) + .withValidity(startTimeUA2, "2099-10-01T12:25Z") + .withClusterType(ClusterType.TARGET) + .withPartition("${cluster.colo}") + .withDataLocation(testBaseDir1 + "/ua2" + MINUTE_DATE_PATTERN + "/") + .build()).toString(); + + feed = FeedMerlin.fromString(feed).addFeedCluster( + new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[2].getClusters().get(0))) + .withRetention("days(1000000)", ActionType.DELETE) + .withValidity("2012-10-01T12:00Z", "2099-01-01T00:00Z") + .withClusterType(ClusterType.SOURCE) + .withPartition("${cluster.colo}") + .withDataLocation(testBaseDir4 + MINUTE_DATE_PATTERN + "/") + .build()) + .toString(); LOGGER.info("feed: " + Util.prettyPrintXml(feed)); @@ -846,16 +886,16 @@ public class PrismFeedReplicationPartitionExpTest extends BaseTestClass { List<Path> ua3OriginalData05ua1 = HadoopUtil .getAllFilesRecursivelyHDFS(cluster3FS, new Path( - testDirWithDateSourcetarget + "05/ua3/ua1")); + testDirWithDateSourceTarget + "05/ua3/ua1")); List<Path> ua3OriginalData10ua1 = HadoopUtil .getAllFilesRecursivelyHDFS(cluster3FS, new Path( - testDirWithDateSourcetarget + "10/ua3/ua1")); + testDirWithDateSourceTarget + "10/ua3/ua1")); List<Path> ua3OriginalData10ua2 = HadoopUtil .getAllFilesRecursivelyHDFS(cluster3FS, new Path( - testDirWithDateSourcetarget + "10/ua3/ua2")); + testDirWithDateSourceTarget + "10/ua3/ua2")); List<Path> ua3OriginalData15ua2 = HadoopUtil .getAllFilesRecursivelyHDFS(cluster3FS, new Path( - testDirWithDateSourcetarget + "15/ua3/ua2")); + testDirWithDateSourceTarget + "15/ua3/ua2")); AssertUtil.checkForListSizes(ua1ReplicatedData05, ua3OriginalData05ua1); AssertUtil.checkForListSizes(ua1ReplicatedData10, ua3OriginalData10ua1); @@ -872,27 +912,33 @@ public class PrismFeedReplicationPartitionExpTest extends BaseTestClass { String startTimeUA2 = "2012-10-01T12:10Z"; String feed = bundles[0].getDataSets().get(0); - feed = InstanceUtil.setFeedCluster(feed, - XmlUtil.createValidity("2012-10-01T12:00Z", "2010-01-01T00:00Z"), - XmlUtil.createRetention("days(1000000)", ActionType.DELETE), null, - ClusterType.SOURCE, null); - - feed = InstanceUtil - .setFeedCluster(feed, XmlUtil.createValidity(startTimeUA1, "2012-10-01T12:10Z"), - XmlUtil.createRetention("days(1000000)", ActionType.DELETE), - Util.readEntityName(bundles[0].getClusters().get(0)), ClusterType.SOURCE, "", - testBaseDir1 + MINUTE_DATE_PATTERN); - - feed = InstanceUtil - .setFeedCluster(feed, XmlUtil.createValidity(startTimeUA2, "2012-10-01T12:25Z"), - XmlUtil.createRetention("days(1000000)", ActionType.DELETE), - Util.readEntityName(bundles[2].getClusters().get(0)), ClusterType.TARGET, "", - testBaseDir2 + MINUTE_DATE_PATTERN); - - feed = InstanceUtil.setFeedCluster(feed, - XmlUtil.createValidity("2012-10-01T12:00Z", "2099-01-01T00:00Z"), - XmlUtil.createRetention("days(1000000)", ActionType.DELETE), - Util.readEntityName(bundles[2].getClusters().get(0)), ClusterType.SOURCE, ""); + feed = FeedMerlin.fromString(feed).clearFeedClusters().toString(); + + feed = FeedMerlin.fromString(feed).addFeedCluster( + new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[0].getClusters().get(0))) + .withRetention("days(1000000)", ActionType.DELETE) + .withValidity(startTimeUA1, "2012-10-01T12:10Z") + .withClusterType(ClusterType.SOURCE) + .withPartition("") + .withDataLocation(testBaseDir1 + MINUTE_DATE_PATTERN) + .build()).toString(); + + feed = FeedMerlin.fromString(feed).addFeedCluster( + new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[2].getClusters().get(0))) + .withRetention("days(1000000)", ActionType.DELETE) + .withValidity(startTimeUA2, "2012-10-01T12:25Z") + .withClusterType(ClusterType.TARGET) + .withPartition("") + .withDataLocation(testBaseDir2 + MINUTE_DATE_PATTERN) + .build()).toString(); + + feed = FeedMerlin.fromString(feed).addFeedCluster( + new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[2].getClusters().get(0))) + .withRetention("days(1000000)", ActionType.DELETE) + .withValidity("2012-10-01T12:00Z", "2099-01-01T00:00Z") + .withClusterType(ClusterType.SOURCE) + .withPartition("") + .build()).toString(); LOGGER.info("feed: " + Util.prettyPrintXml(feed)); http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4f2b524d/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedReplicationUpdateTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedReplicationUpdateTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedReplicationUpdateTest.java index 76345e0..0bd4c31 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedReplicationUpdateTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedReplicationUpdateTest.java @@ -18,6 +18,8 @@ package org.apache.falcon.regression.prism; +import org.apache.falcon.regression.Entities.FeedMerlin; +import org.apache.falcon.regression.Entities.ProcessMerlin; import org.apache.falcon.regression.core.bundle.Bundle; import org.apache.falcon.entity.v0.EntityType; import org.apache.falcon.entity.v0.feed.ActionType; @@ -30,7 +32,6 @@ import org.apache.falcon.regression.core.util.InstanceUtil; import org.apache.falcon.regression.core.util.OSUtil; import org.apache.falcon.regression.core.util.TimeUtil; import org.apache.falcon.regression.core.util.Util; -import org.apache.falcon.regression.core.util.XmlUtil; import org.apache.falcon.regression.testHelper.BaseTestClass; import org.apache.hadoop.fs.FileSystem; import org.apache.log4j.Logger; @@ -104,10 +105,7 @@ public class PrismFeedReplicationUpdateTest extends BaseTestClass { Bundle.submitCluster(bundles[0], bundles[1], bundles[2]); String feed = bundles[0].getDataSets().get(0); - feed = InstanceUtil.setFeedCluster(feed, - XmlUtil.createValidity("2009-02-01T00:00Z", "2012-01-01T00:00Z"), - XmlUtil.createRetention("hours(10)", ActionType.DELETE), null, - ClusterType.SOURCE, null); + feed = FeedMerlin.fromString(feed).clearFeedClusters().toString(); // use the colo string here so that the test works in embedded and distributed mode. String postFix = "/US/" + cluster2Colo; @@ -123,24 +121,31 @@ public class PrismFeedReplicationUpdateTest extends BaseTestClass { String startTime = TimeUtil.getTimeWrtSystemTime(-30); - feed = InstanceUtil.setFeedCluster(feed, XmlUtil.createValidity(startTime, - TimeUtil.addMinsToTime(startTime, 85)), - XmlUtil.createRetention("hours(10)", ActionType.DELETE), - Util.readEntityName(bundles[1].getClusters().get(0)), ClusterType.SOURCE, - "US/${cluster.colo}"); - - feed = InstanceUtil.setFeedCluster(feed, - XmlUtil.createValidity(TimeUtil.addMinsToTime(startTime, 20), - TimeUtil.addMinsToTime(startTime, 105)), - XmlUtil.createRetention("hours(10)", ActionType.DELETE), - Util.readEntityName(bundles[0].getClusters().get(0)), ClusterType.TARGET, null); - - feed = InstanceUtil.setFeedCluster(feed, - XmlUtil.createValidity(TimeUtil.addMinsToTime(startTime, 40), - TimeUtil.addMinsToTime(startTime, 130)), - XmlUtil.createRetention("hours(10)", ActionType.DELETE), - Util.readEntityName(bundles[2].getClusters().get(0)), ClusterType.SOURCE, - "UK/${cluster.colo}"); + feed = FeedMerlin.fromString(feed) + .addFeedCluster(new FeedMerlin.FeedClusterBuilder( + Util.readEntityName(bundles[1].getClusters().get(0))) + .withRetention("hours(10)", ActionType.DELETE) + .withValidity(startTime, TimeUtil.addMinsToTime(startTime, 85)) + .withClusterType(ClusterType.SOURCE) + .withPartition("US/${cluster.colo}") + .build()).toString(); + + feed = FeedMerlin.fromString(feed).addFeedCluster( + new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[0].getClusters().get(0))) + .withRetention("hours(10)", ActionType.DELETE) + .withValidity(TimeUtil.addMinsToTime(startTime, 20), + TimeUtil.addMinsToTime(startTime, 105)) + .withClusterType(ClusterType.TARGET) + .build()).toString(); + + feed = FeedMerlin.fromString(feed).addFeedCluster( + new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[2].getClusters().get(0))) + .withRetention("hours(10)", ActionType.DELETE) + .withValidity(TimeUtil.addMinsToTime(startTime, 40), + TimeUtil.addMinsToTime(startTime, 130)) + .withClusterType(ClusterType.SOURCE) + .withPartition("UK/${cluster.colo}") + .build()).toString(); LOGGER.info("feed: " + Util.prettyPrintXml(feed)); @@ -198,20 +203,9 @@ public class PrismFeedReplicationUpdateTest extends BaseTestClass { String outputFeed = bundles[0].getOutputFeedFromBundle(); //set clusters to null; - feed01 = InstanceUtil.setFeedCluster(feed01, - XmlUtil.createValidity("2009-02-01T00:00Z", "2012-01-01T00:00Z"), - XmlUtil.createRetention("hours(10)", ActionType.DELETE), null, - ClusterType.SOURCE, null); - - feed02 = InstanceUtil.setFeedCluster(feed02, - XmlUtil.createValidity("2009-02-01T00:00Z", "2012-01-01T00:00Z"), - XmlUtil.createRetention("hours(10)", ActionType.DELETE), null, - ClusterType.SOURCE, null); - - outputFeed = InstanceUtil.setFeedCluster(outputFeed, - XmlUtil.createValidity("2009-02-01T00:00Z", "2012-01-01T00:00Z"), - XmlUtil.createRetention("hours(10)", ActionType.DELETE), null, - ClusterType.SOURCE, null); + feed01 = FeedMerlin.fromString(feed01).clearFeedClusters().toString(); + feed02 = FeedMerlin.fromString(feed02).clearFeedClusters().toString(); + outputFeed = FeedMerlin.fromString(outputFeed).clearFeedClusters().toString(); //set new feed input data feed01 = Util.setFeedPathValue(feed01, baseHDFSDir + "/feed01" + MINUTE_DATE_PATTERN); @@ -229,41 +223,49 @@ public class PrismFeedReplicationUpdateTest extends BaseTestClass { String startTime = TimeUtil.getTimeWrtSystemTime(-50); //set clusters for feed01 - feed01 = InstanceUtil - .setFeedCluster(feed01, XmlUtil.createValidity(startTime, "2099-01-01T00:00Z"), - XmlUtil.createRetention("hours(10)", ActionType.DELETE), - Util.readEntityName(bundles[0].getClusters().get(0)), ClusterType.SOURCE, - null); - - feed01 = InstanceUtil - .setFeedCluster(feed01, XmlUtil.createValidity(startTime, "2099-01-01T00:00Z"), - XmlUtil.createRetention("hours(10)", ActionType.DELETE), - Util.readEntityName(bundles[2].getClusters().get(0)), ClusterType.TARGET, - null); + feed01 = FeedMerlin.fromString(feed01).addFeedCluster( + new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[0].getClusters().get(0))) + .withRetention("hours(10)", ActionType.DELETE) + .withValidity(startTime, "2099-01-01T00:00Z") + .withClusterType(ClusterType.SOURCE) + .build()).toString(); + + feed01 = FeedMerlin.fromString(feed01).addFeedCluster( + new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[2].getClusters().get(0))) + .withRetention("hours(10)", ActionType.DELETE) + .withValidity(startTime, "2099-01-01T00:00Z") + .withClusterType(ClusterType.TARGET) + .build()).toString(); //set clusters for feed02 - feed02 = InstanceUtil - .setFeedCluster(feed02, XmlUtil.createValidity(startTime, "2099-01-01T00:00Z"), - XmlUtil.createRetention("hours(10)", ActionType.DELETE), - Util.readEntityName(bundles[0].getClusters().get(0)), ClusterType.TARGET, - null); - - feed02 = InstanceUtil - .setFeedCluster(feed02, XmlUtil.createValidity(startTime, "2099-01-01T00:00Z"), - XmlUtil.createRetention("hours(10)", ActionType.DELETE), - Util.readEntityName(bundles[2].getClusters().get(0)), ClusterType.SOURCE, - null); + feed02 = FeedMerlin.fromString(feed02).addFeedCluster( + new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[0].getClusters().get(0))) + .withRetention("hours(10)", ActionType.DELETE) + .withValidity(startTime, "2099-01-01T00:00Z") + .withClusterType(ClusterType.TARGET) + .build()).toString(); + + feed02 = FeedMerlin.fromString(feed02).addFeedCluster( + new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[2].getClusters().get(0))) + .withRetention("hours(10)", ActionType.DELETE) + .withValidity(startTime, "2099-01-01T00:00Z") + .withClusterType(ClusterType.SOURCE) + .build()).toString(); //set clusters for output feed - outputFeed = InstanceUtil.setFeedCluster(outputFeed, - XmlUtil.createValidity(startTime, "2099-01-01T00:00Z"), - XmlUtil.createRetention("hours(10)", ActionType.DELETE), - Util.readEntityName(bundles[0].getClusters().get(0)), ClusterType.SOURCE, null); - - outputFeed = InstanceUtil.setFeedCluster(outputFeed, - XmlUtil.createValidity(startTime, "2099-01-01T00:00Z"), - XmlUtil.createRetention("hours(10)", ActionType.DELETE), - Util.readEntityName(bundles[2].getClusters().get(0)), ClusterType.TARGET, null); + outputFeed = FeedMerlin.fromString(outputFeed).addFeedCluster( + new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[0].getClusters().get(0))) + .withRetention("hours(10)", ActionType.DELETE) + .withValidity(startTime, "2099-01-01T00:00Z") + .withClusterType(ClusterType.SOURCE) + .build()).toString(); + + outputFeed = FeedMerlin.fromString(outputFeed).addFeedCluster( + new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[2].getClusters().get(0))) + .withRetention("hours(10)", ActionType.DELETE) + .withValidity(startTime, "2099-01-01T00:00Z") + .withClusterType(ClusterType.TARGET) + .build()).toString(); //submit and schedule feeds prism.getFeedHelper().submitAndSchedule(feed01); @@ -279,16 +281,21 @@ public class PrismFeedReplicationUpdateTest extends BaseTestClass { String processStartTime = TimeUtil.getTimeWrtSystemTime(-6); String processEndTime = TimeUtil.getTimeWrtSystemTime(70); - process = InstanceUtil.setProcessCluster(process, null, - XmlUtil.createProcessValidity(startTime, "2099-01-01T00:00Z")); - - process = InstanceUtil - .setProcessCluster(process, Util.readEntityName(bundles[0].getClusters().get(0)), - XmlUtil.createProcessValidity(processStartTime, processEndTime)); - - process = InstanceUtil - .setProcessCluster(process, Util.readEntityName(bundles[2].getClusters().get(0)), - XmlUtil.createProcessValidity(processStartTime, processEndTime)); + process = ProcessMerlin.fromString(process).clearProcessCluster().toString(); + + process = ProcessMerlin.fromString(process).addProcessCluster( + new ProcessMerlin.ProcessClusterBuilder( + Util.readEntityName(bundles[0].getClusters().get(0))) + .withValidity(processStartTime, processEndTime) + .build() + ).toString(); + + process = ProcessMerlin.fromString(process).addProcessCluster( + new ProcessMerlin.ProcessClusterBuilder( + Util.readEntityName(bundles[2].getClusters().get(0))) + .withValidity(processStartTime, processEndTime) + .build() + ).toString(); process = InstanceUtil.addProcessInputFeed(process, Util.readEntityName(feed02), Util.readEntityName(feed02)); http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4f2b524d/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedSnSTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedSnSTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedSnSTest.java index 1cf44b7..3c48fc4 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedSnSTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedSnSTest.java @@ -18,6 +18,7 @@ package org.apache.falcon.regression.prism; +import org.apache.falcon.regression.Entities.FeedMerlin; import org.apache.falcon.regression.core.bundle.Bundle; import org.apache.falcon.entity.v0.EntityType; import org.apache.falcon.entity.v0.feed.ActionType; @@ -26,11 +27,9 @@ import org.apache.falcon.regression.core.helpers.ColoHelper; import org.apache.falcon.regression.core.response.ServiceResponse; import org.apache.falcon.regression.core.util.AssertUtil; import org.apache.falcon.regression.core.util.BundleUtil; -import org.apache.falcon.regression.core.util.InstanceUtil; import org.apache.falcon.regression.core.util.OSUtil; import org.apache.falcon.regression.core.util.OozieUtil; import org.apache.falcon.regression.core.util.Util; -import org.apache.falcon.regression.core.util.XmlUtil; import org.apache.falcon.regression.testHelper.BaseTestClass; import org.apache.log4j.Logger; import org.apache.oozie.client.Job; @@ -384,21 +383,25 @@ public class PrismFeedSnSTest extends BaseTestClass { String startTimeUA2 = "2012-10-01T12:00Z"; String feed = bundles[0].getDataSets().get(0); - feed = InstanceUtil.setFeedCluster(feed, - XmlUtil.createValidity("2012-10-01T12:00Z", "2010-01-01T00:00Z"), - XmlUtil.createRetention("days(10000)", ActionType.DELETE), null, - ClusterType.SOURCE, null); - - feed = InstanceUtil - .setFeedCluster(feed, XmlUtil.createValidity(startTimeUA1, "2099-10-01T12:10Z"), - XmlUtil.createRetention("days(10000)", ActionType.DELETE), - Util.readEntityName(clust1), ClusterType.SOURCE, "${cluster.colo}", - baseHDFSDir + "/localDC/rc/billing" + MINUTE_DATE_PATTERN); - feed = InstanceUtil - .setFeedCluster(feed, XmlUtil.createValidity(startTimeUA2, "2099-10-01T12:25Z"), - XmlUtil.createRetention("days(10000)", ActionType.DELETE), - Util.readEntityName(clust2), ClusterType.TARGET, null, baseHDFSDir - + "/clusterPath/localDC/rc/billing" + MINUTE_DATE_PATTERN); + feed = FeedMerlin.fromString(feed).clearFeedClusters().toString(); + + feed = FeedMerlin.fromString(feed).addFeedCluster( + new FeedMerlin.FeedClusterBuilder(Util.readEntityName(clust1)) + .withRetention("days(10000)", ActionType.DELETE) + .withValidity(startTimeUA1, "2099-10-01T12:10Z") + .withClusterType(ClusterType.SOURCE) + .withPartition("${cluster.colo}") + .withDataLocation(baseHDFSDir + "/localDC/rc/billing" + MINUTE_DATE_PATTERN) + .build()) + .toString(); + feed = FeedMerlin.fromString(feed).addFeedCluster( + new FeedMerlin.FeedClusterBuilder(Util.readEntityName(clust2)) + .withRetention("days(10000)", ActionType.DELETE) + .withValidity(startTimeUA2, "2099-10-01T12:25Z") + .withClusterType(ClusterType.TARGET) + .withDataLocation( + baseHDFSDir + "/clusterPath/localDC/rc/billing" + MINUTE_DATE_PATTERN) + .build()).toString(); LOGGER.info("feed: " + Util.prettyPrintXml(feed)); Util.shutDownService(cluster1.getFeedHelper()); http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4f2b524d/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedUpdateTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedUpdateTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedUpdateTest.java index 902ec23..9f31e84 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedUpdateTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedUpdateTest.java @@ -33,7 +33,6 @@ import org.apache.falcon.regression.core.util.OSUtil; import org.apache.falcon.regression.core.util.OozieUtil; import org.apache.falcon.regression.core.util.TimeUtil; import org.apache.falcon.regression.core.util.Util; -import org.apache.falcon.regression.core.util.XmlUtil; import org.apache.falcon.regression.testHelper.BaseTestClass; import org.apache.hadoop.fs.FileSystem; import org.apache.log4j.Logger; @@ -123,16 +122,8 @@ public class PrismFeedUpdateTest extends BaseTestClass { /* set source and target for the 2 feeds */ //set clusters to null; - feed01 = InstanceUtil - .setFeedCluster(feed01, - XmlUtil.createValidity("2009-02-01T00:00Z", "2012-01-01T00:00Z"), - XmlUtil.createRetention("hours(10)", ActionType.DELETE), null, - ClusterType.SOURCE, null); - outputFeed = InstanceUtil - .setFeedCluster(outputFeed, - XmlUtil.createValidity("2009-02-01T00:00Z", "2012-01-01T00:00Z"), - XmlUtil.createRetention("hours(10)", ActionType.DELETE), null, - ClusterType.SOURCE, null); + feed01 = FeedMerlin.fromString(feed01).clearFeedClusters().toString(); + outputFeed = FeedMerlin.fromString(outputFeed).clearFeedClusters().toString(); //set new feed input data feed01 = Util.setFeedPathValue(feed01, baseTestDir + "/feed01" + MINUTE_DATE_PATTERN); @@ -145,24 +136,32 @@ public class PrismFeedUpdateTest extends BaseTestClass { HadoopUtil.lateDataReplenish(server1FS, 80, 20, prefix, null); //set clusters for feed01 - feed01 = InstanceUtil - .setFeedCluster(feed01, XmlUtil.createValidity(startTime, "2099-01-01T00:00Z"), - XmlUtil.createRetention("hours(10)", ActionType.DELETE), - Util.readEntityName(cluster1Def), ClusterType.SOURCE, null); - feed01 = InstanceUtil - .setFeedCluster(feed01, XmlUtil.createValidity(startTime, "2099-01-01T00:00Z"), - XmlUtil.createRetention("hours(10)", ActionType.DELETE), - Util.readEntityName(cluster2Def), ClusterType.TARGET, null); + feed01 = FeedMerlin.fromString(feed01).addFeedCluster( + new FeedMerlin.FeedClusterBuilder(Util.readEntityName(cluster1Def)) + .withRetention("hours(10)", ActionType.DELETE) + .withValidity(startTime, "2099-01-01T00:00Z") + .withClusterType(ClusterType.SOURCE) + .build()).toString(); + feed01 = FeedMerlin.fromString(feed01).addFeedCluster( + new FeedMerlin.FeedClusterBuilder(Util.readEntityName(cluster2Def)) + .withRetention("hours(10)", ActionType.DELETE) + .withValidity(startTime, "2099-01-01T00:00Z") + .withClusterType(ClusterType.TARGET) + .build()).toString(); //set clusters for output feed - outputFeed = InstanceUtil.setFeedCluster(outputFeed, - XmlUtil.createValidity(startTime, "2099-01-01T00:00Z"), - XmlUtil.createRetention("hours(10)", ActionType.DELETE), - Util.readEntityName(cluster1Def), ClusterType.SOURCE, null); - outputFeed = InstanceUtil.setFeedCluster(outputFeed, - XmlUtil.createValidity(startTime, "2099-01-01T00:00Z"), - XmlUtil.createRetention("hours(10)", ActionType.DELETE), - Util.readEntityName(cluster2Def), ClusterType.TARGET, null); + outputFeed = FeedMerlin.fromString(outputFeed).addFeedCluster( + new FeedMerlin.FeedClusterBuilder(Util.readEntityName(cluster1Def)) + .withRetention("hours(10)", ActionType.DELETE) + .withValidity(startTime, "2099-01-01T00:00Z") + .withClusterType(ClusterType.SOURCE) + .build()).toString(); + outputFeed = FeedMerlin.fromString(outputFeed).addFeedCluster( + new FeedMerlin.FeedClusterBuilder(Util.readEntityName(cluster2Def)) + .withRetention("hours(10)", ActionType.DELETE) + .withValidity(startTime, "2099-01-01T00:00Z") + .withClusterType(ClusterType.TARGET) + .build()).toString(); //submit and schedule feeds LOGGER.info("feed01: " + Util.prettyPrintXml(feed01)); @@ -177,15 +176,17 @@ public class PrismFeedUpdateTest extends BaseTestClass { //add clusters to process String processStartTime = TimeUtil.getTimeWrtSystemTime(-11); String processEndTime = TimeUtil.getTimeWrtSystemTime(70); - process01 = InstanceUtil - .setProcessCluster(process01, null, - XmlUtil.createProcessValidity(startTime, "2099-01-01T00:00Z")); - process01 = InstanceUtil - .setProcessCluster(process01, Util.readEntityName(cluster1Def), - XmlUtil.createProcessValidity(processStartTime, processEndTime)); - process01 = InstanceUtil - .setProcessCluster(process01, Util.readEntityName(cluster2Def), - XmlUtil.createProcessValidity(processStartTime, processEndTime)); + process01 = ProcessMerlin.fromString(process01).clearProcessCluster().toString(); + process01 = ProcessMerlin.fromString(process01).addProcessCluster( + new ProcessMerlin.ProcessClusterBuilder(Util.readEntityName(cluster1Def)) + .withValidity(processStartTime, processEndTime) + .build() + ).toString(); + process01 = ProcessMerlin.fromString(process01).addProcessCluster( + new ProcessMerlin.ProcessClusterBuilder(Util.readEntityName(cluster2Def)) + .withValidity(processStartTime, processEndTime) + .build() + ).toString(); //get 2nd process String process02 = process01; http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4f2b524d/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismSubmitTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismSubmitTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismSubmitTest.java index 279dc56..f176da7 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismSubmitTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismSubmitTest.java @@ -80,7 +80,7 @@ public class PrismSubmitTest extends BaseTestClass { } @Test(groups = "distributed") - public void submitCluster1prism1coloPrismDown() throws Exception { + public void submitCluster1Prism1ColoPrismDown() throws Exception { restartRequired = true; Util.shutDownService(prism.getClusterHelper()); @@ -98,7 +98,7 @@ public class PrismSubmitTest extends BaseTestClass { } @Test(groups = "distributed") - public void submitClusterResubmitDiffContent() throws Exception { + public void submitClusterReSubmitDiffContent() throws Exception { ServiceResponse r = prism.getClusterHelper().submitEntity(bundles[0].getClusters().get(0)); Assert.assertTrue(r.getMessage().contains("SUCCEEDED")); List<String> beforeSubmitCluster = cluster1.getClusterHelper().getStoreInfo(); @@ -117,7 +117,7 @@ public class PrismSubmitTest extends BaseTestClass { } @Test(groups = "distributed") - public void submitClusterResubmitAlreadyPARTIALWithAllUp() throws Exception { + public void submitClusterResubmitAlreadyPartialWithAllUp() throws Exception { restartRequired = true; Util.shutDownService(cluster1.getClusterHelper()); TimeUtil.sleepSeconds(30); @@ -231,7 +231,7 @@ public class PrismSubmitTest extends BaseTestClass { } @Test(groups = "distributed") - public void submitCluster1prism1coloColoDown() throws Exception { + public void submitCluster1Prism1ColoColoDown() throws Exception { restartRequired = true; Util.shutDownService(cluster1.getClusterHelper()); @@ -271,7 +271,7 @@ public class PrismSubmitTest extends BaseTestClass { } @Test(groups = "distributed") - public void submitCluster1prism1coloSubmitDeleted() throws Exception { + public void submitCluster1Prism1ColoSubmitDeleted() throws Exception { prism.getClusterHelper().submitEntity(bundles[0].getClusters().get(0)); prism.getClusterHelper().delete(bundles[0].getClusters().get(0)); @@ -296,7 +296,7 @@ public class PrismSubmitTest extends BaseTestClass { } @Test(groups = "embedded") - public void submitProcessWOClusterSubmit() throws Exception { + public void submitProcessWoClusterSubmit() throws Exception { ServiceResponse r = prism.getProcessHelper().submitEntity(bundles[0].getProcessData()); Assert.assertTrue(r.getMessage().contains("FAILED")); @@ -304,7 +304,7 @@ public class PrismSubmitTest extends BaseTestClass { } @Test(groups = "embedded") - public void submitProcessWOFeedSubmit() throws Exception { + public void submitProcessWoFeedSubmit() throws Exception { ServiceResponse r = prism.getClusterHelper().submitEntity(bundles[0].getClusters().get(0)); Assert.assertTrue(r.getMessage().contains("SUCCEEDED")); @@ -314,7 +314,7 @@ public class PrismSubmitTest extends BaseTestClass { } @Test(groups = {"prism", "0.2", "distributed"}) - public void submitClusterResubmitAlreadyPARTIAL() throws Exception { + public void submitClusterReSubmitAlreadyPartial() throws Exception { restartRequired = true; bundles[1] = new Bundle(bundles[0], cluster2); bundles[1].generateUniqueBundle(); @@ -388,7 +388,7 @@ public class PrismSubmitTest extends BaseTestClass { } @Test(groups = "distributed") - public void submitClusterResubmitDiffContentPARTIAL() throws Exception { + public void submitClusterResubmitDiffContentPartial() throws Exception { restartRequired = true; Util.shutDownService(cluster1.getClusterHelper()); TimeUtil.sleepSeconds(30); @@ -412,7 +412,7 @@ public class PrismSubmitTest extends BaseTestClass { } @Test - public void submitClusterPARTIALDeletedOfPARTIALSubmit() throws Exception { + public void submitClusterPartialDeletedOfPartialSubmit() throws Exception { restartRequired = true; Util.shutDownService(cluster1.getClusterHelper()); TimeUtil.sleepSeconds(30); @@ -470,7 +470,7 @@ public class PrismSubmitTest extends BaseTestClass { } @Test(groups = "embedded") - public void submitClusterResubmitAlreadySucceeded() throws Exception { + public void submitClusterReSubmitAlreadySucceeded() throws Exception { ServiceResponse r = prism.getClusterHelper().submitEntity(bundles[0].getClusters().get(0)); Assert.assertTrue(r.getMessage().contains("SUCCEEDED")); @@ -486,7 +486,7 @@ public class PrismSubmitTest extends BaseTestClass { } @Test(groups = "distributed") - public void submitCluster1prism1coloAllUp() throws Exception { + public void submitCluster1Prism1ColoAllUp() throws Exception { List<String> beforeSubmitCluster1 = cluster1.getClusterHelper().getStoreInfo(); List<String> beforeSubmitCluster2 = cluster2.getClusterHelper().getStoreInfo(); List<String> beforeSubmitPrism = prism.getClusterHelper().getStoreInfo(); @@ -507,7 +507,7 @@ public class PrismSubmitTest extends BaseTestClass { } @Test(groups = "embedded") - public void submitCluster1prism1coloAlreadySubmitted() throws Exception { + public void submitCluster1Prism1ColoAlreadySubmitted() throws Exception { prism.getClusterHelper().submitEntity(bundles[0].getClusters().get(0)); List<String> beforeSubmitCluster1 = cluster1.getClusterHelper().getStoreInfo(); http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4f2b524d/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RescheduleKilledProcessTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RescheduleKilledProcessTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RescheduleKilledProcessTest.java index 4a0c10e..32c9783 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RescheduleKilledProcessTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RescheduleKilledProcessTest.java @@ -27,7 +27,6 @@ import org.apache.falcon.regression.core.util.InstanceUtil; import org.apache.falcon.regression.core.util.OSUtil; import org.apache.falcon.regression.core.util.TimeUtil; import org.apache.falcon.regression.core.util.Util; -import org.apache.falcon.regression.core.util.XmlUtil; import org.apache.falcon.regression.testHelper.BaseTestClass; import org.apache.log4j.Logger; import org.testng.annotations.AfterClass; @@ -88,11 +87,13 @@ public class RescheduleKilledProcessTest extends BaseTestClass { processMerlin.setProcessFeeds(feed, 0, 0, 1); process = processMerlin.toString(); - process = InstanceUtil.setProcessCluster(process, null, - XmlUtil.createProcessValidity(processStartTime, "2099-01-01T00:00Z")); - process = InstanceUtil - .setProcessCluster(process, Util.readEntityName(bundles[0].getClusters().get(0)), - XmlUtil.createProcessValidity(processStartTime, processEndTime)); + process = ProcessMerlin.fromString(process).clearProcessCluster().toString(); + process = ProcessMerlin.fromString(process).addProcessCluster( + new ProcessMerlin.ProcessClusterBuilder( + Util.readEntityName(bundles[0].getClusters().get(0))) + .withValidity(processStartTime, processEndTime) + .build() + ).toString(); bundles[0].setProcessData(process); bundles[0].submitFeedsScheduleProcess(prism); http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4f2b524d/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ui/LineageGraphTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ui/LineageGraphTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ui/LineageGraphTest.java index 51fcb8d..cd32ea4 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ui/LineageGraphTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ui/LineageGraphTest.java @@ -210,8 +210,8 @@ public class LineageGraphTest extends BaseUITestClass { Assert.assertEquals(info.get("Runs on"), clusterName, String.format(message, "cluster", processName)); } - Assert.assertEquals(info.get("Owned by"), System.getProperty("user" - + ".name"), "Entity should be owned by current system user."); + Assert.assertEquals(info.get("Owned by"), System.getProperty("user.name"), + "Entity should be owned by current system user."); } } processPage.refresh(); @@ -300,7 +300,7 @@ public class LineageGraphTest extends BaseUITestClass { List<Edge> incEdges = lineageHelper.getEdgesByDirection(piVertex.getId(), Direction.inComingEdges).getResults(); List<Edge> outcEdges = lineageHelper.getEdgesByDirection(piVertex.getId(), - Direction.outGoingEdges).filterByType(Edge.LEBEL_TYPE.OUTPUT); + Direction.outGoingEdges).filterByType(Edge.LabelType.OUTPUT); assert expectedEdgesAPI.addAll(incEdges); assert expectedEdgesAPI.addAll(outcEdges); /** Check the number of edges and their location*/ @@ -328,9 +328,9 @@ public class LineageGraphTest extends BaseUITestClass { break; } } - Assert.assertTrue( - isEdgePresent, String.format("Edge %s-->%s isn't present on lineage or " - + "painted incorrectly.", startVertexAPI.getName(), endVertexAPI.getName())); + Assert.assertTrue(isEdgePresent, + String.format("Edge %s-->%s isn't present on lineage or painted incorrectly.", + startVertexAPI.getName(), endVertexAPI.getName())); } processPage.refresh(); } http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4f2b524d/falcon-regression/pom.xml ---------------------------------------------------------------------- diff --git a/falcon-regression/pom.xml b/falcon-regression/pom.xml index c204d14..be17513 100644 --- a/falcon-regression/pom.xml +++ b/falcon-regression/pom.xml @@ -34,13 +34,12 @@ <packaging>pom</packaging> <modules> - <module>checkstyle</module> <module>merlin-core</module> <module>merlin</module> </modules> <properties> - <oozie.version>3.2.2</oozie.version> + <oozie.version>4.0.0.2.1.7.0-784</oozie.version> <hive.version>0.13.1</hive.version> </properties> @@ -359,6 +358,7 @@ <exclude>*.txt</exclude> <exclude>.git/**</exclude> <exclude>.idea/**</exclude> + <exclude>oozie_logs/**</exclude> <exclude>**/*.twiki</exclude> <exclude>**/*.iml</exclude> <exclude>**/target/**</exclude> @@ -427,7 +427,7 @@ <!--debug>true</debug --> <xmlOutput>true</xmlOutput> <excludeFilterFile> - ${project.basedir}/../checkstyle/src/main/resources/falcon/findbugs-exclude.xml + ${project.basedir}/../../checkstyle/src/main/resources/falcon/findbugs-exclude.xml </excludeFilterFile> <failOnError>true</failOnError> <skip>${skipCheck}</skip>
