Repository: falcon Updated Branches: refs/heads/master 65cb71841 -> 9dd0b27e5
FALCON-1043 Add test in falcon to test touch feature. Contributed by Pragya M. Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/87bff1fc Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/87bff1fc Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/87bff1fc Branch: refs/heads/master Commit: 87bff1fc111f0fb58324db82a6734fa73cefcbc1 Parents: 65cb718 Author: samarthg <[email protected]> Authored: Wed Feb 25 10:51:17 2015 +0530 Committer: samarthg <[email protected]> Committed: Wed Feb 25 10:51:17 2015 +0530 ---------------------------------------------------------------------- falcon-regression/CHANGES.txt | 2 + .../helpers/entity/AbstractEntityHelper.java | 12 ++ .../falcon/regression/core/util/Util.java | 4 +- .../regression/TouchAPIPrismAndServerTest.java | 183 +++++++++++++++++++ 4 files changed, 200 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/87bff1fc/falcon-regression/CHANGES.txt ---------------------------------------------------------------------- diff --git a/falcon-regression/CHANGES.txt b/falcon-regression/CHANGES.txt index 4ad8a2a..fc3ae50 100644 --- a/falcon-regression/CHANGES.txt +++ b/falcon-regression/CHANGES.txt @@ -6,6 +6,8 @@ Trunk (Unreleased) NEW FEATURES + FALCON-1043 Add test in falcon to test touch feature(Pragya Mittal via Samarth Gupta) + FALCON-1021 Add test in falcon to test feed SLA feature (Pragya Mittal via Raghav Kumar Gautam) FALCON-671 Add a test in falcon-regression where one oozie workflow has various actions like http://git-wip-us.apache.org/repos/asf/falcon/blob/87bff1fc/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java index e11b37d..3828b2f 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java @@ -597,4 +597,16 @@ public abstract class AbstractEntityHelper { entityName + colo); return Util.sendRequest(url, "get", null, null); } + + public ServiceResponse touchEntity(String data) + throws IOException, URISyntaxException, AuthenticationException, InterruptedException { + return touchEntity(Util.readEntityName(data), data, null); + } + + public ServiceResponse touchEntity(String entityName, String data, String user) + throws AuthenticationException, IOException, URISyntaxException, InterruptedException { + String url = createUrl(this.hostname + URLS.TOUCH_URL.getValue(), getEntityType(), + entityName + colo); + return Util.sendRequest(url, "post", data, user); + } } http://git-wip-us.apache.org/repos/asf/falcon/blob/87bff1fc/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/Util.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/Util.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/Util.java index 5717923..ab09ff1 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/Util.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/Util.java @@ -596,7 +596,9 @@ public final class Util { INSTANCE_RERUN("/api/instance/rerun"), INSTANCE_SUMMARY("/api/instance/summary"), INSTANCE_PARAMS("/api/instance/params"), - INSTANCE_LIST("/api/instance/list"); + INSTANCE_LIST("/api/instance/list"), + TOUCH_URL("/api/entities/touch"); + private final String url; URLS(String url) { http://git-wip-us.apache.org/repos/asf/falcon/blob/87bff1fc/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TouchAPIPrismAndServerTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TouchAPIPrismAndServerTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TouchAPIPrismAndServerTest.java new file mode 100644 index 0000000..1f3d990 --- /dev/null +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TouchAPIPrismAndServerTest.java @@ -0,0 +1,183 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.falcon.regression; + +import org.apache.falcon.entity.v0.EntityType; +import org.apache.falcon.entity.v0.Frequency; +import org.apache.falcon.regression.core.bundle.Bundle; +import org.apache.falcon.regression.core.helpers.ColoHelper; +import org.apache.falcon.regression.core.response.ServiceResponse; +import org.apache.falcon.regression.core.util.*; +import org.apache.falcon.regression.testHelper.BaseTestClass; +import org.apache.hadoop.fs.FileSystem; +import org.apache.log4j.Logger; +import org.apache.oozie.client.Job; +import org.apache.oozie.client.OozieClient; +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import javax.xml.bind.JAXBException; + +/** + * Touch feature test both via server and prism. + */ +public class TouchAPIPrismAndServerTest extends BaseTestClass { + private ColoHelper cluster = servers.get(0); + private OozieClient clusterOC = serverOC.get(0); + private FileSystem clusterFS = serverFS.get(0); + private String aggregateWorkflowDir = cleanAndGetTestDir() + "/aggregator"; + private String feed; + private static final Logger LOGGER = Logger.getLogger(TouchAPIPrismAndServerTest.class); + private String startTime; + private String endTime; + + @BeforeClass(alwaysRun = true) + public void uploadWorkflow() throws Exception { + uploadDirToClusters(aggregateWorkflowDir, OSUtil.RESOURCES_OOZIE); + } + + @BeforeMethod(alwaysRun = true) + public void setUp() throws Exception { + startTime = TimeUtil.getTimeWrtSystemTime(0); + endTime = TimeUtil.addMinsToTime(startTime, 20); + LOGGER.info("Time range between : " + startTime + " and " + endTime); + bundles[0] = BundleUtil.readELBundle(); + bundles[0] = new Bundle(bundles[0], cluster); + bundles[0].generateUniqueBundle(this); + bundles[0].setProcessWorkflow(aggregateWorkflowDir); + bundles[0].setProcessValidity(startTime, endTime); + bundles[0].setProcessPeriodicity(5, Frequency.TimeUnit.minutes); + bundles[0].setOutputFeedPeriodicity(5, Frequency.TimeUnit.minutes); + } + + @AfterMethod(alwaysRun = true) + public void tearDown() { + removeTestClassEntities(); + } + + /** + * Schedule a process with end time greater than current time + * Perform touch via both server and prism + * Should succeed with creation of a new bundle from the current time. + * + * @throws Exception + */ + @Test(groups = {"singleCluster"}) + public void touchProcessSchedule() throws Exception { + bundles[0].submitFeedsScheduleProcess(prism); + AssertUtil.checkStatus(clusterOC, EntityType.PROCESS, bundles[0], Job.Status.RUNNING); + InstanceUtil.waitTillInstancesAreCreated(cluster, bundles[0].getProcessData(), 0); + String coordId = InstanceUtil.getLatestCoordinatorID(clusterOC, + bundles[0].getProcessName(), EntityType.PROCESS); + String oldbundleId = InstanceUtil.getLatestBundleID(cluster, bundles[0].getProcessName(), EntityType.PROCESS); + + // via prism + ServiceResponse response = prism.getProcessHelper().touchEntity(bundles[0].getProcessData()); + String bundleId = InstanceUtil.getLatestBundleID(cluster, bundles[0].getProcessName(), EntityType.PROCESS); + Assert.assertNotEquals(oldbundleId, bundleId, "Bundle ids are same. No new bundle generated."); + validate(response, "Old bundle id: " + coordId + ". New bundle id: " + bundleId); + + // via server + oldbundleId = bundleId; + coordId = InstanceUtil.getLatestCoordinatorID(clusterOC, bundles[0].getProcessName(), EntityType.PROCESS); + response = cluster.getProcessHelper().touchEntity(bundles[0].getProcessData()); + bundleId = InstanceUtil.getLatestBundleID(cluster, bundles[0].getProcessName(), EntityType.PROCESS); + Assert.assertNotEquals(oldbundleId, bundleId, "Bundle ids are same. No new bundle generated."); + validate(response, "Old bundle id: " + coordId + ". New bundle id: " + bundleId); + } + + /** + * Schedule a feed with end time greater than current time + * Perform touch via both server and prism + * Should succeed with creation of a new bundle from the current time. + * + * @throws Exception + */ + @Test(groups = {"singleCluster"}) + public void touchFeedSchedule() throws Exception { + bundles[0].submitAndScheduleFeed(); + AssertUtil.checkStatus(clusterOC, EntityType.FEED, bundles[0], Job.Status.RUNNING); + String coordId = InstanceUtil.getLatestCoordinatorID(clusterOC, + Util.readEntityName(bundles[0].getDataSets().get(0)), EntityType.FEED); + String oldbundleId = InstanceUtil.getLatestBundleID(cluster, + Util.readEntityName(bundles[0].getDataSets().get(0)), EntityType.FEED); + + // via prism + TimeUtil.sleepSeconds(60); + ServiceResponse response = prism.getFeedHelper().touchEntity(bundles[0].getDataSets().get(0)); + String bundleId = InstanceUtil.getLatestBundleID(cluster, Util.readEntityName(bundles[0].getDataSets().get(0)), + EntityType.FEED); + Assert.assertNotEquals(oldbundleId, bundleId, "Bundle ids are same. No new bundle generated."); + validate(response, "Old bundle id: " + coordId + ". New bundle id: " + bundleId); + + // via server + oldbundleId = bundleId; + coordId = InstanceUtil.getLatestCoordinatorID(clusterOC, Util.readEntityName(bundles[0].getDataSets().get(0)), + EntityType.FEED); + TimeUtil.sleepSeconds(60); + response = cluster.getFeedHelper().touchEntity(bundles[0].getDataSets().get(0)); + bundleId = InstanceUtil.getLatestBundleID(cluster, Util.readEntityName(bundles[0].getDataSets().get(0)), + EntityType.FEED); + Assert.assertNotEquals(oldbundleId, bundleId, "Bundle ids are same. No new bundle generated."); + validate(response, "Old bundle id: " + coordId + ". New bundle id: " + bundleId); + + } + + /** + * Schedule a process with end time less than current time. + * Perform touch via both server and prism + * No new bundle should be created. + * + * @throws Exception + */ + @Test(groups = {"singleCluster"}) + public void touchProcessScheduleWithEndTimeLessThanCurrentTime() throws Exception { + startTime = TimeUtil.getTimeWrtSystemTime(-120); + endTime = TimeUtil.addMinsToTime(startTime, 20); + bundles[0].setProcessValidity(startTime, endTime); + bundles[0].submitFeedsScheduleProcess(prism); + AssertUtil.checkStatus(clusterOC, EntityType.PROCESS, bundles[0], Job.Status.RUNNING); + InstanceUtil.waitTillInstancesAreCreated(cluster, bundles[0].getProcessData(), 0); + String coordId = InstanceUtil.getLatestCoordinatorID(clusterOC, + bundles[0].getProcessName(), EntityType.PROCESS); + String oldbundleId = InstanceUtil.getLatestBundleID(cluster, bundles[0].getProcessName(), EntityType.PROCESS); + + // via prism + ServiceResponse response = prism.getProcessHelper().touchEntity(bundles[0].getProcessData()); + String bundleId = InstanceUtil.getLatestBundleID(cluster, bundles[0].getProcessName(), EntityType.PROCESS); + Assert.assertEquals(oldbundleId, bundleId, "New bundle generated"); + validate(response, "Old bundle id: " + coordId); + + //via server + oldbundleId = bundleId; + response = cluster.getProcessHelper().touchEntity(bundles[0].getProcessData()); + bundleId = InstanceUtil.getLatestBundleID(cluster, bundles[0].getProcessName(), EntityType.PROCESS); + Assert.assertEquals(oldbundleId, bundleId, "New bundle generated"); + validate(response, "Old bundle id: " + coordId); + } + + private void validate(ServiceResponse response, String message) throws JAXBException { + AssertUtil.assertSucceeded(response); + Assert.assertTrue(response.getMessage().contains(message), + "Correct response was not present in process / feed schedule"); + } +}
