Repository: falcon Updated Branches: refs/heads/master 1685541f7 -> c23eaf7cc
FALCON-1021 Add test in falcon to test feed SLA feature. Contributed by Pragya Mittal Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/c23eaf7c Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/c23eaf7c Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/c23eaf7c Branch: refs/heads/master Commit: c23eaf7cc04e42d21fe766dcd8a757acd4171c00 Parents: 1685541 Author: Raghav Kumar Gautam <[email protected]> Authored: Tue Feb 17 14:26:41 2015 -0800 Committer: Raghav Kumar Gautam <[email protected]> Committed: Tue Feb 17 14:26:41 2015 -0800 ---------------------------------------------------------------------- falcon-regression/CHANGES.txt | 2 + .../falcon/regression/Entities/FeedMerlin.java | 14 ++ .../apache/falcon/regression/FeedSLATest.java | 212 +++++++++++++++++++ 3 files changed, 228 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/c23eaf7c/falcon-regression/CHANGES.txt ---------------------------------------------------------------------- diff --git a/falcon-regression/CHANGES.txt b/falcon-regression/CHANGES.txt index a88fb85..fcf04c4 100644 --- a/falcon-regression/CHANGES.txt +++ b/falcon-regression/CHANGES.txt @@ -6,6 +6,8 @@ Trunk (Unreleased) NEW FEATURES + FALCON-1021 Add test in falcon to test feed SLA feature (Pragya Mittal via Raghav Kumar Gautam) + FALCON-671 Add a test in falcon-regression where one oozie workflow has various actions like mr, hive, pig (Karishma Gulati via Ruslan Ostafiychuk) http://git-wip-us.apache.org/repos/asf/falcon/blob/c23eaf7c/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/FeedMerlin.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/FeedMerlin.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/FeedMerlin.java index 47eb1bd..a458de9 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/FeedMerlin.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/FeedMerlin.java @@ -35,6 +35,7 @@ import org.apache.falcon.entity.v0.feed.Locations; import org.apache.falcon.entity.v0.feed.Property; import org.apache.falcon.entity.v0.feed.Retention; import org.apache.falcon.entity.v0.feed.Validity; +import org.apache.falcon.entity.v0.feed.Sla; import org.apache.falcon.regression.core.util.TimeUtil; import org.testng.Assert; @@ -244,4 +245,17 @@ public class FeedMerlin extends Feed { this.setACL(acl); } + /** + * Sel SLA. + * @param slaLow : low value of SLA + * @param slaHigh : high value of SLA + */ + + public void setSla(Frequency slaLow, Frequency slaHigh) { + Sla sla = new Sla(); + sla.setSlaLow(slaLow); + sla.setSlaHigh(slaHigh); + this.setSla(sla); + } + } http://git-wip-us.apache.org/repos/asf/falcon/blob/c23eaf7c/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSLATest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSLATest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSLATest.java new file mode 100644 index 0000000..89d6d87 --- /dev/null +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSLATest.java @@ -0,0 +1,212 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.falcon.regression; + +import org.apache.falcon.entity.v0.Frequency; +import org.apache.falcon.entity.v0.feed.ActionType; +import org.apache.falcon.regression.Entities.FeedMerlin; +import org.apache.falcon.regression.core.bundle.Bundle; +import org.apache.falcon.regression.core.helpers.ColoHelper; +import org.apache.falcon.regression.core.response.ServiceResponse; +import org.apache.falcon.regression.core.util.*; +import org.apache.falcon.regression.testHelper.BaseTestClass; +import org.apache.log4j.Logger; +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + + +/** + * Feed SLA tests. + */ +@Test(groups = "embedded") +public class FeedSLATest extends BaseTestClass { + + private ColoHelper cluster = servers.get(0); + private String baseTestDir = cleanAndGetTestDir(); + private String feedInputPath = baseTestDir + "/input" + MINUTE_DATE_PATTERN; + private static final Logger LOGGER = Logger.getLogger(FeedSLATest.class); + + private FeedMerlin feedMerlin; + private String startTime; + private String endTime; + + @BeforeMethod(alwaysRun = true) + public void setup() throws Exception { + Bundle bundle = BundleUtil.readELBundle(); + bundles[0] = new Bundle(bundle, cluster); + bundles[0].generateUniqueBundle(this); + bundles[0].setInputFeedDataPath(feedInputPath); + + startTime = TimeUtil.getTimeWrtSystemTime(0); + endTime = TimeUtil.addMinsToTime(startTime, 120); + LOGGER.info("Time range between : " + startTime + " and " + endTime); + ServiceResponse response = + prism.getClusterHelper().submitEntity(bundles[0].getClusters().get(0)); + AssertUtil.assertSucceeded(response); + + feedMerlin = new FeedMerlin(bundles[0].getInputFeedFromBundle()); + feedMerlin.setFrequency(new Frequency("1", Frequency.TimeUnit.hours)); + } + + @AfterMethod(alwaysRun = true) + public void tearDown() { + removeTestClassEntities(); + } + + /** + * Submit feed with correctly adjusted sla. Response should reflect success. + * + */ + + @Test + public void submitValidFeedSLA() throws Exception { + + feedMerlin.clearFeedClusters(); + feedMerlin.addFeedCluster(new FeedMerlin.FeedClusterBuilder( + Util.readEntityName(bundles[0].getClusters().get(0))) + .withRetention("days(1000000)", ActionType.DELETE) + .withValidity(startTime, endTime) + .build()); + + //set slaLow and slaHigh + feedMerlin.setSla(new Frequency("3", Frequency.TimeUnit.hours), new Frequency("6", Frequency.TimeUnit.hours)); + + final ServiceResponse serviceResponse = + prism.getFeedHelper().submitEntity(feedMerlin.toString()); + AssertUtil.assertSucceeded(serviceResponse); + } + + /** + * Submit feed with slaHigh greater than feed retention. Response should reflect failure. + * + */ + + @Test + public void submitFeedWithSLAHigherThanRetention() throws Exception { + + feedMerlin.clearFeedClusters(); + feedMerlin.addFeedCluster(new FeedMerlin.FeedClusterBuilder( + Util.readEntityName(bundles[0].getClusters().get(0))) + .withRetention((new Frequency("2", Frequency.TimeUnit.hours)).toString(), ActionType.DELETE) + .withValidity(startTime, endTime) + .build()); + + //set slaLow and slaHigh + feedMerlin.setSla(new Frequency("3", Frequency.TimeUnit.hours), new Frequency("6", Frequency.TimeUnit.hours)); + + final ServiceResponse serviceResponse = + prism.getFeedHelper().submitEntity(feedMerlin.toString()); + String message = "Feed's retention limit: " + + feedMerlin.getClusters().getClusters().get(0).getRetention().getLimit() + + " of referenced cluster " + bundles[0].getClusterNames().get(0) + + " should be more than feed's late arrival cut-off period: " + + feedMerlin.getSla().getSlaHigh().getTimeUnit() + + "(" + feedMerlin.getSla().getSlaHigh().getFrequency() + ")" + + " for feed: " + bundles[0].getInputFeedNameFromBundle(); + validate(serviceResponse, message); + } + + + /** + * Submit feed with slaHigh less than slaLow. Response should reflect failure. + * + */ + @Test + public void submitFeedWithSLAHighLowerthanSLALow() throws Exception { + + feedMerlin.clearFeedClusters(); + feedMerlin.addFeedCluster(new FeedMerlin.FeedClusterBuilder( + Util.readEntityName(bundles[0].getClusters().get(0))) + .withRetention((new Frequency("6", Frequency.TimeUnit.hours)).toString(), ActionType.DELETE) + .withValidity(startTime, endTime) + .build()); + + //set slaLow and slaHigh + feedMerlin.setSla(new Frequency("4", Frequency.TimeUnit.hours), new Frequency("2", Frequency.TimeUnit.hours)); + + final ServiceResponse serviceResponse = + prism.getFeedHelper().submitEntity(feedMerlin.toString()); + String message = "slaLow of Feed: " + feedMerlin.getSla().getSlaLow().getTimeUnit() + "(" + + feedMerlin.getSla().getSlaLow().getFrequency() + ")is greater than slaHigh: " + + feedMerlin.getSla().getSlaHigh().getTimeUnit() + "(" + feedMerlin.getSla().getSlaHigh().getFrequency() + + ") for cluster: " + bundles[0].getClusterNames().get(0); + validate(serviceResponse, message); + } + + /** + * Submit feed with slaHigh and slaLow greater than feed retention. Response should reflect failure. + * + */ + @Test + public void submitFeedWithSLAHighSLALowHigherThanRetention() throws Exception { + + feedMerlin.clearFeedClusters(); + feedMerlin.addFeedCluster(new FeedMerlin.FeedClusterBuilder( + Util.readEntityName(bundles[0].getClusters().get(0))) + .withRetention((new Frequency("4", Frequency.TimeUnit.hours)).toString(), ActionType.DELETE) + .withValidity(startTime, endTime) + .build()); + + //set slaLow and slaHigh + feedMerlin.setSla(new Frequency("5", Frequency.TimeUnit.hours), new Frequency("6", Frequency.TimeUnit.hours)); + + final ServiceResponse serviceResponse = + prism.getFeedHelper().submitEntity(feedMerlin.toString()); + String message = "Feed's retention limit: " + + feedMerlin.getClusters().getClusters().get(0).getRetention().getLimit() + + " of referenced cluster " + bundles[0].getClusterNames().get(0) + + " should be more than feed's late arrival cut-off period: " + + feedMerlin.getSla().getSlaHigh().getTimeUnit() +"(" + feedMerlin.getSla().getSlaHigh().getFrequency() + + ")" + " for feed: " + bundles[0].getInputFeedNameFromBundle(); + validate(serviceResponse, message); + } + + /** + * Submit feed with slaHigh and slaLow having equal value. Response should reflect success. + * + */ + @Test + public void submitFeedWithSameSLAHighSLALow() throws Exception { + + feedMerlin.clearFeedClusters(); + feedMerlin.addFeedCluster(new FeedMerlin.FeedClusterBuilder( + Util.readEntityName(bundles[0].getClusters().get(0))) + .withRetention((new Frequency("7", Frequency.TimeUnit.hours)).toString(), ActionType.DELETE) + .withValidity(startTime, endTime) + .build()); + + //set slaLow and slaHigh + feedMerlin.setSla(new Frequency("3", Frequency.TimeUnit.hours), new Frequency("3", Frequency.TimeUnit.hours)); + + final ServiceResponse serviceResponse = + prism.getFeedHelper().submitEntity(feedMerlin.toString()); + AssertUtil.assertSucceeded(serviceResponse); + } + + private void validate(ServiceResponse response, String message) throws Exception { + AssertUtil.assertFailed(response); + LOGGER.info("Expected message is : " + message); + Assert.assertTrue(response.getMessage().contains(message), + "Correct response was not present in feed schedule. Feed response is : " + + response.getMessage()); + } + +}
