Repository: falcon Updated Branches: refs/heads/master 4b523130f -> 62fb403be
FALCON-1635 Migration of EntityManagerJerseySmokeIT to use falcon unit. Contributed by Narayan Periwal. Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/94a70585 Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/94a70585 Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/94a70585 Branch: refs/heads/master Commit: 94a70585d73a09c8efcf2f134cf32b1d11cd2a6b Parents: 4b52313 Author: Ajay Yadava <[email protected]> Authored: Thu Dec 17 19:14:06 2015 +0530 Committer: Ajay Yadava <[email protected]> Committed: Thu Dec 17 19:14:06 2015 +0530 ---------------------------------------------------------------------- CHANGES.txt | 2 + .../AbstractSchedulerManagerJerseyIT.java | 116 +++++++++++++- .../falcon/resource/AbstractTestContext.java | 69 ++++++++ .../falcon/resource/EntityManagerJerseyIT.java | 156 +------------------ .../resource/EntityManagerJerseySmokeIT.java | 87 +++++------ .../EntitySchedulerManagerJerseyIT.java | 8 + .../InstanceSchedulerManagerJerseyIT.java | 9 +- .../resource/ProcessInstanceManagerIT.java | 48 +----- .../org/apache/falcon/resource/TestContext.java | 29 +--- .../apache/falcon/resource/UnitTestContext.java | 43 +---- 10 files changed, 256 insertions(+), 311 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/94a70585/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 37c5d67..ae45515 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -35,6 +35,8 @@ Trunk (Unreleased) FALCON-1213 Base framework of the native scheduler(Pallavi Rao) IMPROVEMENTS + FALCON-1635 Migration of EntityManagerJerseySmokeIT to use falcon unit(Narayan Periwal via Ajay Yadava) + FALCON-1447 Integration Tests for native scheduler(Pavan Kumar Kolamuri via Ajay Yadava) FALCON-1617 Enable SLA monitoring for instances in past(Narayan Periwal via Ajay Yadava) http://git-wip-us.apache.org/repos/asf/falcon/blob/94a70585/webapp/src/test/java/org/apache/falcon/resource/AbstractSchedulerManagerJerseyIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/falcon/resource/AbstractSchedulerManagerJerseyIT.java b/webapp/src/test/java/org/apache/falcon/resource/AbstractSchedulerManagerJerseyIT.java index f5bcc54..5001fe6 100644 --- a/webapp/src/test/java/org/apache/falcon/resource/AbstractSchedulerManagerJerseyIT.java +++ b/webapp/src/test/java/org/apache/falcon/resource/AbstractSchedulerManagerJerseyIT.java @@ -19,30 +19,42 @@ package org.apache.falcon.resource; import org.apache.commons.lang3.StringUtils; +import org.apache.falcon.FalconException; import org.apache.falcon.client.FalconCLIException; import org.apache.falcon.entity.v0.EntityType; +import org.apache.falcon.hadoop.HadoopClientFactory; import org.apache.falcon.state.AbstractSchedulerTestBase; import org.apache.falcon.state.store.service.FalconJPAService; import org.apache.falcon.unit.FalconUnitTestBase; import org.apache.falcon.util.StartupProperties; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FsShell; import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Path; +import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import java.io.File; import java.io.IOException; +import java.text.DateFormat; +import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; +import java.util.Date; import java.util.List; import java.util.Map; +import java.util.TimeZone; /** * Base class for tests using Native Scheduler. */ public class AbstractSchedulerManagerJerseyIT extends FalconUnitTestBase { + private static final String SLEEP_WORKFLOW = "sleepWorkflow.xml"; + private static final String LOCAL_MODE = "local"; + private static final String IT_RUN_MODE = "it.run.mode"; + public static final String PROCESS_TEMPLATE = "/local-process-noinputs-template.xml"; public static final String PROCESS_NAME = "processName"; protected static final String START_INSTANCE = "2012-04-20T00:00Z"; @@ -54,19 +66,19 @@ public class AbstractSchedulerManagerJerseyIT extends FalconUnitTestBase { protected LocalFileSystem localFS = new LocalFileSystem(); + @BeforeClass public void setup() throws Exception { Configuration localConf = new Configuration(); localFS.initialize(LocalFileSystem.getDefaultUri(localConf), localConf); cleanupDB(); localFS.mkdirs(new Path(DB_BASE_DIR)); - updateStartUpProps(); falconJPAService.init(); createDB(); super.setup(); } - private void updateStartUpProps() { + protected void updateStartUpProps() { StartupProperties.get().setProperty("workflow.engine.impl", "org.apache.falcon.workflow.engine.FalconWorkflowEngine"); StartupProperties.get().setProperty("dag.engine.impl", @@ -120,4 +132,104 @@ public class AbstractSchedulerManagerJerseyIT extends FalconUnitTestBase { private void cleanupDB() throws IOException { localFS.delete(new Path(DB_BASE_DIR), true); } + + protected void submitCluster(UnitTestContext context) throws IOException, FalconCLIException { + String mode = System.getProperty(IT_RUN_MODE); + if (StringUtils.isNotEmpty(mode) && mode.toLowerCase().equals(LOCAL_MODE)) { + submitCluster(context.colo, context.clusterName, null); + } else { + fs.mkdirs(new Path(STAGING_PATH), HadoopClientFactory.ALL_PERMISSION); + fs.mkdirs(new Path(WORKING_PATH), HadoopClientFactory.READ_EXECUTE_PERMISSION); + String tmpFile = TestContext.overlayParametersOverTemplate(TestContext.CLUSTER_TEMPLATE, + context.overlay); + submit(EntityType.CLUSTER, tmpFile); + } + } + + protected APIResult submitFeed(String template, Map<String, String> overlay) throws IOException, + FalconCLIException { + String tmpFile = TestContext.overlayParametersOverTemplate(template, overlay); + APIResult result = falconUnitClient.submit(EntityType.FEED.name(), tmpFile, null); + return result; + } + + protected void submitFeeds(Map<String, String> overlay) throws IOException, FalconCLIException { + String tmpFile = TestContext.overlayParametersOverTemplate(UnitTestContext.FEED_TEMPLATE1, overlay); + APIResult result = falconUnitClient.submit(EntityType.FEED.name(), tmpFile, null); + Assert.assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED); + tmpFile = TestContext.overlayParametersOverTemplate(UnitTestContext.FEED_TEMPLATE2, overlay); + result = falconUnitClient.submit(EntityType.FEED.name(), tmpFile, null); + Assert.assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED); + } + + protected void submitProcess(String template, Map<String, String> overlay) throws Exception { + String tmpFile = TestContext.overlayParametersOverTemplate(template, overlay); + APIResult result = falconUnitClient.submit(EntityType.PROCESS.name(), tmpFile, null); + Assert.assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED); + } + + protected void scheduleProcess(UnitTestContext context) throws FalconCLIException, IOException, FalconException { + String scheduleTime = START_INSTANCE; + APIResult result = scheduleProcess(context.getProcessName(), scheduleTime, 1, context.getClusterName(), + getAbsolutePath(SLEEP_WORKFLOW), true, ""); + Assert.assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED); + } + + protected void schedule(UnitTestContext context) throws Exception { + submitCluster(context); + context.prepare(); + submitFeeds(context.overlay); + submitProcess(UnitTestContext.PROCESS_TEMPLATE, context.overlay); + scheduleProcess(context); + } + + protected List<Path> createTestData() throws Exception { + List<Path> list = new ArrayList<Path>(); + fs.mkdirs(new Path("/user/guest")); + fs.setOwner(new Path("/user/guest"), TestContext.REMOTE_USER, "users"); + + DateFormat formatter = new SimpleDateFormat("yyyy/MM/dd/HH/mm"); + formatter.setTimeZone(TimeZone.getTimeZone("UTC")); + Date date = new Date(System.currentTimeMillis() + 3 * 3600000); + Path path = new Path("/examples/input-data/rawLogs/" + formatter.format(date) + "/file"); + fs.create(path).close(); + date = new Date(date.getTime() - 3600000); + path = new Path("/examples/input-data/rawLogs/" + formatter.format(date) + "/file"); + fs.create(path).close(); + date = new Date(date.getTime() - 3600000); + path = new Path("/examples/input-data/rawLogs/" + formatter.format(date) + "/file"); + fs.create(path).close(); + date = new Date(date.getTime() - 3600000); + path = new Path("/examples/input-data/rawLogs/" + formatter.format(date) + "/file"); + list.add(path); + fs.create(path).close(); + date = new Date(date.getTime() - 3600000); + path = new Path("/examples/input-data/rawLogs/" + formatter.format(date) + "/file"); + list.add(path); + fs.create(path).close(); + date = new Date(date.getTime() - 3600000); + path = new Path("/examples/input-data/rawLogs/" + formatter.format(date) + "/file"); + list.add(path); + fs.create(path).close(); + date = new Date(date.getTime() - 3600000); + path = new Path("/examples/input-data/rawLogs/" + formatter.format(date) + "/file"); + list.add(path); + fs.create(path).close(); + date = new Date(date.getTime() - 3600000); + path = new Path("/examples/input-data/rawLogs/" + formatter.format(date) + "/file"); + list.add(path); + fs.create(path).close(); + date = new Date(date.getTime() - 3600000); + path = new Path("/examples/input-data/rawLogs/" + formatter.format(date) + "/file"); + list.add(path); + fs.create(path).close(); + date = new Date(date.getTime() - 3600000); + path = new Path("/examples/input-data/rawLogs/" + formatter.format(date) + "/file"); + list.add(path); + fs.create(path).close(); + new FsShell(new Configuration()).run(new String[] { + "-chown", "-R", "guest:users", "/examples/input-data/rawLogs", }); + return list; + } + } http://git-wip-us.apache.org/repos/asf/falcon/blob/94a70585/webapp/src/test/java/org/apache/falcon/resource/AbstractTestContext.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/falcon/resource/AbstractTestContext.java b/webapp/src/test/java/org/apache/falcon/resource/AbstractTestContext.java new file mode 100644 index 0000000..413dfde --- /dev/null +++ b/webapp/src/test/java/org/apache/falcon/resource/AbstractTestContext.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.falcon.resource; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; + +import java.io.File; +import java.io.IOException; + +/** + * Base class for context for unit and integration tests. + */ +public abstract class AbstractTestContext { + + public static final String FEED_TEMPLATE1 = "/feed-template1.xml"; + public static final String FEED_TEMPLATE2 = "/feed-template2.xml"; + public static final String FEED_TEMPLATE3 = "/feed-template3.xml"; + + public static final String PROCESS_TEMPLATE = "/process-template.xml"; + + protected static void mkdir(FileSystem fileSystem, Path path) throws Exception { + if (!fileSystem.exists(path) && !fileSystem.mkdirs(path)) { + throw new Exception("mkdir failed for " + path); + } + } + + protected static void mkdir(FileSystem fileSystem, Path path, FsPermission perm) throws Exception { + if (!fileSystem.exists(path) && !fileSystem.mkdirs(path, perm)) { + throw new Exception("mkdir failed for " + path); + } + } + + public static File getTempFile() throws IOException { + return getTempFile("test", ".xml"); + } + + public static File getTempFile(String prefix, String suffix) throws IOException { + return getTempFile("target", prefix, suffix); + } + + @SuppressWarnings("ResultOfMethodCallIgnored") + public static File getTempFile(String path, String prefix, String suffix) throws IOException { + File f = new File(path); + if (!f.exists()) { + f.mkdirs(); + } + + return File.createTempFile(prefix, suffix, f); + } + +} http://git-wip-us.apache.org/repos/asf/falcon/blob/94a70585/webapp/src/test/java/org/apache/falcon/resource/EntityManagerJerseyIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/falcon/resource/EntityManagerJerseyIT.java b/webapp/src/test/java/org/apache/falcon/resource/EntityManagerJerseyIT.java index 258bb1a..7157766 100644 --- a/webapp/src/test/java/org/apache/falcon/resource/EntityManagerJerseyIT.java +++ b/webapp/src/test/java/org/apache/falcon/resource/EntityManagerJerseyIT.java @@ -19,7 +19,6 @@ package org.apache.falcon.resource; import com.sun.jersey.api.client.ClientResponse; import org.apache.commons.io.FileUtils; -import org.apache.commons.lang.StringUtils; import org.apache.falcon.FalconException; import org.apache.falcon.client.FalconCLIException; import org.apache.falcon.entity.v0.Entity; @@ -35,15 +34,12 @@ import org.apache.falcon.entity.v0.process.Property; import org.apache.falcon.entity.v0.process.Validity; import org.apache.falcon.hadoop.HadoopClientFactory; import org.apache.falcon.unit.FalconUnit; -import org.apache.falcon.unit.FalconUnitTestBase; import org.apache.falcon.util.BuildProperties; import org.apache.falcon.util.DeploymentProperties; import org.apache.falcon.util.FalconTestUtil; import org.apache.falcon.util.OozieTestUtils; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FsShell; import org.apache.hadoop.fs.Path; import org.apache.oozie.client.BundleJob; import org.apache.oozie.client.CoordinatorJob; @@ -58,9 +54,6 @@ import org.testng.annotations.Test; import javax.ws.rs.core.MediaType; import java.io.File; import java.io.IOException; -import java.text.DateFormat; -import java.text.SimpleDateFormat; -import java.util.ArrayList; import java.util.Calendar; import java.util.Date; import java.util.HashMap; @@ -80,12 +73,10 @@ import java.util.regex.Pattern; * Tests should be enabled only in local environments as they need running instance of the web server. */ @Test(groups = {"exhaustive"}) -public class EntityManagerJerseyIT extends FalconUnitTestBase { +public class EntityManagerJerseyIT extends AbstractSchedulerManagerJerseyIT { private static final String START_INSTANCE = "2012-04-20T00:00Z"; private static final String SLEEP_WORKFLOW = "sleepWorkflow.xml"; - private static final String LOCAL_MODE = "local"; - private static final String IT_RUN_MODE = "it.run.mode"; @BeforeClass @Override @@ -127,52 +118,6 @@ public class EntityManagerJerseyIT extends FalconUnitTestBase { processValidity.setEnd(new Date(new Date().getTime() + 2 * 24 * 60 * 60 * 1000)); } - private void submitCluster(UnitTestContext context) throws IOException, FalconCLIException { - String mode = System.getProperty(IT_RUN_MODE); - if (StringUtils.isNotEmpty(mode) && mode.toLowerCase().equals(LOCAL_MODE)) { - submitCluster(context.colo, context.clusterName, null); - } else { - String tmpFile = TestContext.overlayParametersOverTemplate(TestContext.CLUSTER_TEMPLATE, context.overlay); - submit(EntityType.CLUSTER, tmpFile); - } - } - - private APIResult submitFeed(String template, Map<String, String> overlay) throws IOException, FalconCLIException { - String tmpFile = TestContext.overlayParametersOverTemplate(template, overlay); - APIResult result = falconUnitClient.submit(EntityType.FEED.name(), tmpFile, null); - return result; - } - - private void submitProcess(String template, Map<String, String> overlay) throws IOException, FalconCLIException { - String tmpFile = TestContext.overlayParametersOverTemplate(template, overlay); - APIResult result = falconUnitClient.submit(EntityType.PROCESS.name(), tmpFile, null); - Assert.assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED); - } - - private void scheduleProcess(UnitTestContext context) throws FalconCLIException, IOException, FalconException { - String scheduleTime = START_INSTANCE; - APIResult result = scheduleProcess(context.getProcessName(), scheduleTime, 1, context.getClusterName(), - getAbsolutePath(SLEEP_WORKFLOW), true, ""); - Assert.assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED); - } - - private void schedule(UnitTestContext context) throws Exception { - submitCluster(context); - context.prepare(); - submitFeeds(context.overlay); - submitProcess(UnitTestContext.PROCESS_TEMPLATE, context.overlay); - scheduleProcess(context); - } - - private void submitFeeds(Map<String, String> overlay) throws IOException, FalconCLIException { - String tmpFile = TestContext.overlayParametersOverTemplate(UnitTestContext.FEED_TEMPLATE1, overlay); - APIResult result = falconUnitClient.submit(EntityType.FEED.name(), tmpFile, null); - Assert.assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED); - tmpFile = TestContext.overlayParametersOverTemplate(UnitTestContext.FEED_TEMPLATE2, overlay); - result = falconUnitClient.submit(EntityType.FEED.name(), tmpFile, null); - Assert.assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED); - } - @Test public void testLibExtensions() throws Exception { UnitTestContext context = newContext(); @@ -731,105 +676,6 @@ public class EntityManagerJerseyIT extends FalconUnitTestBase { Assert.assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED); } - static List<Path> createTestData() throws Exception { - List<Path> list = new ArrayList<Path>(); - fs.mkdirs(new Path("/user/guest")); - fs.setOwner(new Path("/user/guest"), TestContext.REMOTE_USER, "users"); - - DateFormat formatter = new SimpleDateFormat("yyyy/MM/dd/HH/mm"); - formatter.setTimeZone(TimeZone.getTimeZone("UTC")); - Date date = new Date(System.currentTimeMillis() + 3 * 3600000); - Path path = new Path("/examples/input-data/rawLogs/" + formatter.format(date) + "/file"); - fs.create(path).close(); - date = new Date(date.getTime() - 3600000); - path = new Path("/examples/input-data/rawLogs/" + formatter.format(date) + "/file"); - fs.create(path).close(); - date = new Date(date.getTime() - 3600000); - path = new Path("/examples/input-data/rawLogs/" + formatter.format(date) + "/file"); - fs.create(path).close(); - date = new Date(date.getTime() - 3600000); - path = new Path("/examples/input-data/rawLogs/" + formatter.format(date) + "/file"); - list.add(path); - fs.create(path).close(); - date = new Date(date.getTime() - 3600000); - path = new Path("/examples/input-data/rawLogs/" + formatter.format(date) + "/file"); - list.add(path); - fs.create(path).close(); - date = new Date(date.getTime() - 3600000); - path = new Path("/examples/input-data/rawLogs/" + formatter.format(date) + "/file"); - list.add(path); - fs.create(path).close(); - date = new Date(date.getTime() - 3600000); - path = new Path("/examples/input-data/rawLogs/" + formatter.format(date) + "/file"); - list.add(path); - fs.create(path).close(); - date = new Date(date.getTime() - 3600000); - path = new Path("/examples/input-data/rawLogs/" + formatter.format(date) + "/file"); - list.add(path); - fs.create(path).close(); - date = new Date(date.getTime() - 3600000); - path = new Path("/examples/input-data/rawLogs/" + formatter.format(date) + "/file"); - list.add(path); - fs.create(path).close(); - date = new Date(date.getTime() - 3600000); - path = new Path("/examples/input-data/rawLogs/" + formatter.format(date) + "/file"); - list.add(path); - fs.create(path).close(); - new FsShell(new Configuration()).run(new String[] { - "-chown", "-R", "guest:users", "/examples/input-data/rawLogs", }); - return list; - } - - static List<Path> createTestData(TestContext context) throws Exception { - List<Path> list = new ArrayList<Path>(); - FileSystem fs = context.cluster.getFileSystem(); - fs.mkdirs(new Path("/user/guest")); - fs.setOwner(new Path("/user/guest"), TestContext.REMOTE_USER, "users"); - - DateFormat formatter = new SimpleDateFormat("yyyy/MM/dd/HH/mm"); - formatter.setTimeZone(TimeZone.getTimeZone("UTC")); - Date date = new Date(System.currentTimeMillis() + 3 * 3600000); - Path path = new Path("/examples/input-data/rawLogs/" + formatter.format(date) + "/file"); - fs.create(path).close(); - date = new Date(date.getTime() - 3600000); - path = new Path("/examples/input-data/rawLogs/" + formatter.format(date) + "/file"); - fs.create(path).close(); - date = new Date(date.getTime() - 3600000); - path = new Path("/examples/input-data/rawLogs/" + formatter.format(date) + "/file"); - fs.create(path).close(); - date = new Date(date.getTime() - 3600000); - path = new Path("/examples/input-data/rawLogs/" + formatter.format(date) + "/file"); - list.add(path); - fs.create(path).close(); - date = new Date(date.getTime() - 3600000); - path = new Path("/examples/input-data/rawLogs/" + formatter.format(date) + "/file"); - list.add(path); - fs.create(path).close(); - date = new Date(date.getTime() - 3600000); - path = new Path("/examples/input-data/rawLogs/" + formatter.format(date) + "/file"); - list.add(path); - fs.create(path).close(); - date = new Date(date.getTime() - 3600000); - path = new Path("/examples/input-data/rawLogs/" + formatter.format(date) + "/file"); - list.add(path); - fs.create(path).close(); - date = new Date(date.getTime() - 3600000); - path = new Path("/examples/input-data/rawLogs/" + formatter.format(date) + "/file"); - list.add(path); - fs.create(path).close(); - date = new Date(date.getTime() - 3600000); - path = new Path("/examples/input-data/rawLogs/" + formatter.format(date) + "/file"); - list.add(path); - fs.create(path).close(); - date = new Date(date.getTime() - 3600000); - path = new Path("/examples/input-data/rawLogs/" + formatter.format(date) + "/file"); - list.add(path); - fs.create(path).close(); - new FsShell(context.cluster.getConf()). - run(new String[]{"-chown", "-R", "guest:users", "/examples/input-data/rawLogs"}); - return list; - } - public void testDeleteDataSet() throws Exception { UnitTestContext context = newContext(); http://git-wip-us.apache.org/repos/asf/falcon/blob/94a70585/webapp/src/test/java/org/apache/falcon/resource/EntityManagerJerseySmokeIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/falcon/resource/EntityManagerJerseySmokeIT.java b/webapp/src/test/java/org/apache/falcon/resource/EntityManagerJerseySmokeIT.java index 2bedb88..a3c874d 100644 --- a/webapp/src/test/java/org/apache/falcon/resource/EntityManagerJerseySmokeIT.java +++ b/webapp/src/test/java/org/apache/falcon/resource/EntityManagerJerseySmokeIT.java @@ -17,7 +17,8 @@ */ package org.apache.falcon.resource; -import com.sun.jersey.api.client.ClientResponse; +import org.apache.commons.io.FileUtils; +import org.apache.falcon.FalconException; import org.apache.falcon.entity.v0.EntityType; import org.apache.falcon.entity.v0.process.Process; import org.apache.falcon.entity.v0.process.Property; @@ -29,8 +30,8 @@ import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -import javax.ws.rs.core.MediaType; import java.io.File; +import java.io.IOException; import java.util.List; import java.util.Map; @@ -40,11 +41,17 @@ import java.util.Map; * Tests should be enabled only in local environments as they need running instance of the web server. */ @Test -public class EntityManagerJerseySmokeIT { +public class EntityManagerJerseySmokeIT extends AbstractSchedulerManagerJerseyIT { + + private UnitTestContext context; @BeforeClass - public void prepare() throws Exception { - TestContext.prepare(); + @Override + public void setup() throws Exception { + String version = System.getProperty("project.version"); + String buildDir = System.getProperty("project.build.directory"); + System.setProperty("falcon.libext", buildDir + "/../../unit/target/falcon-unit-" + version + ".jar"); + super.setup(); } @AfterClass @@ -52,28 +59,27 @@ public class EntityManagerJerseySmokeIT { TestContext.deleteEntitiesFromStore(); } - private ThreadLocal<TestContext> contexts = new ThreadLocal<TestContext>(); + private ThreadLocal<UnitTestContext> contexts = new ThreadLocal<UnitTestContext>(); - private TestContext newContext() { - contexts.set(new TestContext()); + private UnitTestContext newContext() throws FalconException, IOException { + contexts.set(new UnitTestContext()); return contexts.get(); } @AfterMethod - public void cleanup() throws Exception { - TestContext testContext = contexts.get(); - if (testContext != null) { - OozieTestUtils.killOozieJobs(testContext); - } + @Override + public void cleanUpActionXml() throws IOException, FalconException { + //Needed since oozie writes action xml to current directory. + FileUtils.deleteQuietly(new File("action.xml")); + FileUtils.deleteQuietly(new File(".action.xml.crc")); contexts.remove(); } @Test (dependsOnMethods = "testFeedSchedule") public void testProcessDeleteAndSchedule() throws Exception { //Schedule process, delete and then submitAndSchedule again should create new bundle - TestContext context = newContext(); - Map<String, String> overlay = context.getUniqueOverlay(); - String tmpFileName = TestContext.overlayParametersOverTemplate(TestContext.PROCESS_TEMPLATE, overlay); + String tmpFileName = TestContext.overlayParametersOverTemplate(UnitTestContext.PROCESS_TEMPLATE, + context.overlay); Process process = (Process) EntityType.PROCESS.getUnmarshaller().unmarshal(new File(tmpFileName)); Property prop = new Property(); prop.setName("newProp"); @@ -81,25 +87,20 @@ public class EntityManagerJerseySmokeIT { process.getProperties().getProperties().add(prop); File tmpFile = TestContext.getTempFile(); EntityType.PROCESS.getMarshaller().marshal(process, tmpFile); - context.scheduleProcess(tmpFile.getAbsolutePath(), overlay); + context.prepare(); + submitProcess(tmpFile.getAbsolutePath(), context.overlay); + scheduleProcess(context); - //Delete and re-submit the process with correct workflow - ClientResponse clientResponse = context.service - .path("api/entities/delete/process/" + context.processName) - .header("Cookie", context.getAuthenticationToken()) - .accept(MediaType.TEXT_XML) - .delete(ClientResponse.class); - context.assertSuccessful(clientResponse); + APIResult result = falconUnitClient.delete(EntityType.PROCESS, context.getProcessName(), null); + Assert.assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED); process.getWorkflow().setPath("/falcon/test/workflow"); tmpFile = TestContext.getTempFile(); EntityType.PROCESS.getMarshaller().marshal(process, tmpFile); - clientResponse = context.service.path("api/entities/submitAndSchedule/process") - .header("Cookie", context.getAuthenticationToken()) - .accept(MediaType.TEXT_XML).type(MediaType.TEXT_XML) - .post(ClientResponse.class, context.getServletInputStream(tmpFile.getAbsolutePath())); - context.assertSuccessful(clientResponse); + falconUnitClient.submitAndSchedule(EntityType.PROCESS.name(), tmpFile.getAbsolutePath(), true, + null, null); + Assert.assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED); //Assert that new schedule creates new bundle List<BundleJob> bundles = OozieTestUtils.getBundles(context); @@ -108,22 +109,16 @@ public class EntityManagerJerseySmokeIT { @Test public void testFeedSchedule() throws Exception { - TestContext context = newContext(); - ClientResponse response; - Map<String, String> overlay = context.getUniqueOverlay(); - - response = context.submitToFalcon(TestContext.CLUSTER_TEMPLATE, overlay, EntityType.CLUSTER); - context.assertSuccessful(response); - - response = context.submitToFalcon(TestContext.FEED_TEMPLATE1, overlay, EntityType.FEED); - context.assertSuccessful(response); - - EntityManagerJerseyIT.createTestData(context); - ClientResponse clientRepsonse = context.service - .path("api/entities/schedule/feed/" + overlay.get("inputFeedName")) - .header("Cookie", context.getAuthenticationToken()) - .accept(MediaType.TEXT_XML).type(MediaType.TEXT_XML) - .post(ClientResponse.class); - context.assertSuccessful(clientRepsonse); + context = newContext(); + Map<String, String> overlay = context.overlay; + + submitCluster(context); + + submitFeed(UnitTestContext.FEED_TEMPLATE1, overlay); + submitFeed(UnitTestContext.FEED_TEMPLATE2, overlay); + + createTestData(); + APIResult result = schedule(EntityType.FEED, context.getInputFeedName(), context.getClusterName()); + Assert.assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED); } } http://git-wip-us.apache.org/repos/asf/falcon/blob/94a70585/webapp/src/test/java/org/apache/falcon/resource/EntitySchedulerManagerJerseyIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/falcon/resource/EntitySchedulerManagerJerseyIT.java b/webapp/src/test/java/org/apache/falcon/resource/EntitySchedulerManagerJerseyIT.java index 35119f0..67a51d9 100644 --- a/webapp/src/test/java/org/apache/falcon/resource/EntitySchedulerManagerJerseyIT.java +++ b/webapp/src/test/java/org/apache/falcon/resource/EntitySchedulerManagerJerseyIT.java @@ -19,6 +19,7 @@ package org.apache.falcon.resource; import org.apache.falcon.entity.v0.EntityType; import org.testng.Assert; +import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; import java.util.Map; @@ -29,6 +30,13 @@ import java.util.Map; */ public class EntitySchedulerManagerJerseyIT extends AbstractSchedulerManagerJerseyIT { + + @BeforeClass + public void setup() throws Exception { + updateStartUpProps(); + super.setup(); + } + @Test public void testEntitySubmitAndSchedule() throws Exception { UnitTestContext context = new UnitTestContext(); http://git-wip-us.apache.org/repos/asf/falcon/blob/94a70585/webapp/src/test/java/org/apache/falcon/resource/InstanceSchedulerManagerJerseyIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/falcon/resource/InstanceSchedulerManagerJerseyIT.java b/webapp/src/test/java/org/apache/falcon/resource/InstanceSchedulerManagerJerseyIT.java index 7959b63..1523b76 100644 --- a/webapp/src/test/java/org/apache/falcon/resource/InstanceSchedulerManagerJerseyIT.java +++ b/webapp/src/test/java/org/apache/falcon/resource/InstanceSchedulerManagerJerseyIT.java @@ -18,13 +18,11 @@ package org.apache.falcon.resource; -import org.apache.falcon.FalconException; -import org.apache.falcon.client.FalconCLIException; import org.apache.falcon.entity.v0.EntityType; import org.testng.Assert; +import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -import java.io.IOException; import java.util.Map; /** @@ -36,6 +34,11 @@ public class InstanceSchedulerManagerJerseyIT extends AbstractSchedulerManagerJe private static final String END_TIME = "2012-04-21T00:00Z"; private static final String HELLO_WORLD_WORKFLOW = "helloworldworkflow.xml"; + @BeforeClass + public void setup() throws Exception { + updateStartUpProps(); + super.setup(); + } @Test public void testProcessInstanceExecution() throws Exception { http://git-wip-us.apache.org/repos/asf/falcon/blob/94a70585/webapp/src/test/java/org/apache/falcon/resource/ProcessInstanceManagerIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/falcon/resource/ProcessInstanceManagerIT.java b/webapp/src/test/java/org/apache/falcon/resource/ProcessInstanceManagerIT.java index f94bd8c..755efa3 100644 --- a/webapp/src/test/java/org/apache/falcon/resource/ProcessInstanceManagerIT.java +++ b/webapp/src/test/java/org/apache/falcon/resource/ProcessInstanceManagerIT.java @@ -20,12 +20,10 @@ package org.apache.falcon.resource; import org.apache.commons.io.FileUtils; import org.apache.falcon.FalconException; -import org.apache.falcon.client.FalconCLIException; import org.apache.falcon.entity.v0.EntityType; import org.apache.falcon.resource.InstancesResult.Instance; import org.apache.falcon.resource.InstancesResult.WorkflowStatus; import org.apache.falcon.security.CurrentUser; -import org.apache.falcon.unit.FalconUnitTestBase; import org.apache.falcon.util.OozieTestUtils; import org.testng.Assert; import org.testng.annotations.AfterMethod; @@ -35,12 +33,11 @@ import org.testng.annotations.Test; import javax.ws.rs.core.MediaType; import java.io.File; import java.io.IOException; -import java.util.Map; /** * Test class for Process Instance REST API. */ -public class ProcessInstanceManagerIT extends FalconUnitTestBase { +public class ProcessInstanceManagerIT extends AbstractSchedulerManagerJerseyIT { private static final String START_INSTANCE = "2012-04-20T00:00Z"; private static final String SLEEP_WORKFLOW = "sleepWorkflow.xml"; @@ -62,37 +59,6 @@ public class ProcessInstanceManagerIT extends FalconUnitTestBase { FileUtils.deleteQuietly(new File(".action.xml.crc")); } - private void submitFeeds(Map<String, String> overlay) throws IOException, FalconCLIException { - String tmpFile = TestContext.overlayParametersOverTemplate(UnitTestContext.FEED_TEMPLATE1, overlay); - APIResult result = falconUnitClient.submit(EntityType.FEED.name(), tmpFile, null); - Assert.assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED); - tmpFile = TestContext.overlayParametersOverTemplate(UnitTestContext.FEED_TEMPLATE2, overlay); - result = falconUnitClient.submit(EntityType.FEED.name(), tmpFile, null); - Assert.assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED); - } - - - private void submitProcess(String template, Map<String, String> overlay) throws IOException, FalconCLIException { - String tmpFile = TestContext.overlayParametersOverTemplate(template, overlay); - APIResult result = falconUnitClient.submit(EntityType.PROCESS.name(), tmpFile, null); - Assert.assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED); - } - - private void schedule(UnitTestContext context) throws FalconCLIException, IOException, FalconException { - String scheduleTime = START_INSTANCE; - APIResult result = scheduleProcess(context.getProcessName(), scheduleTime, 1, context.getClusterName(), - getAbsolutePath(SLEEP_WORKFLOW), true, ""); - Assert.assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED); - } - - private void scheduleProcess(UnitTestContext context) throws Exception { - submitCluster(context.colo, context.clusterName, null); - context.prepare(); - submitFeeds(context.overlay); - submitProcess(UnitTestContext.PROCESS_TEMPLATE, context.overlay); - schedule(context); - } - protected void schedule(TestContext context) throws Exception { CurrentUser.authenticate(System.getProperty("user.name")); schedule(context, 1); @@ -154,7 +120,7 @@ public class ProcessInstanceManagerIT extends FalconUnitTestBase { @Test public void testGetInstanceStatus() throws Exception { UnitTestContext context = new UnitTestContext(); - scheduleProcess(context); + schedule(context); waitForStatus(EntityType.PROCESS.name(), context.processName, START_INSTANCE, WorkflowStatus.RUNNING); String endTime = "2012-04-20T00:01Z"; InstancesResult response = context.getClient().getStatusOfInstances(EntityType.PROCESS.name(), @@ -168,7 +134,7 @@ public class ProcessInstanceManagerIT extends FalconUnitTestBase { @Test public void testGetInstanceStatusPagination() throws Exception { UnitTestContext context = new UnitTestContext(); - scheduleProcess(context); + schedule(context); waitForStatus(EntityType.PROCESS.name(), context.processName, START_INSTANCE, WorkflowStatus.RUNNING); String endTime = "2012-04-20T00:02Z"; InstancesResult response = context.getClient().getStatusOfInstances(EntityType.PROCESS.name(), @@ -183,7 +149,7 @@ public class ProcessInstanceManagerIT extends FalconUnitTestBase { @Test public void testKillInstances() throws Exception { UnitTestContext context = new UnitTestContext(); - scheduleProcess(context); + schedule(context); waitForStatus(EntityType.PROCESS.name(), context.processName, START_INSTANCE, WorkflowStatus.RUNNING); String endTime = "2012-04-20T00:01Z"; context.getClient().killInstances(EntityType.PROCESS.name(), context.processName, START_INSTANCE, endTime, @@ -209,7 +175,7 @@ public class ProcessInstanceManagerIT extends FalconUnitTestBase { @Test public void testReRunInstances() throws Exception { UnitTestContext context = new UnitTestContext(); - scheduleProcess(context); + schedule(context); waitForStatus(EntityType.PROCESS.name(), context.processName, START_INSTANCE, WorkflowStatus.RUNNING); String endTime = "2012-04-20T00:01Z"; context.getClient().killInstances(EntityType.PROCESS.name(), context.processName, START_INSTANCE, endTime, @@ -238,7 +204,7 @@ public class ProcessInstanceManagerIT extends FalconUnitTestBase { @Test public void testSuspendInstances() throws Exception { UnitTestContext context = new UnitTestContext(); - scheduleProcess(context); + schedule(context); waitForStatus(EntityType.PROCESS.name(), context.processName, START_INSTANCE, WorkflowStatus.RUNNING); String endTime = "2012-04-20T00:01Z"; context.getClient().suspendInstances(EntityType.PROCESS.name(), context.processName, START_INSTANCE, @@ -257,7 +223,7 @@ public class ProcessInstanceManagerIT extends FalconUnitTestBase { @Test public void testResumesInstances() throws Exception { UnitTestContext context = new UnitTestContext(); - scheduleProcess(context); + schedule(context); waitForStatus(EntityType.PROCESS.name(), context.processName, START_INSTANCE, WorkflowStatus.RUNNING); String endTime = "2012-04-20T00:01Z"; context.getClient().suspendInstances(EntityType.PROCESS.name(), context.processName, START_INSTANCE, http://git-wip-us.apache.org/repos/asf/falcon/blob/94a70585/webapp/src/test/java/org/apache/falcon/resource/TestContext.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/falcon/resource/TestContext.java b/webapp/src/test/java/org/apache/falcon/resource/TestContext.java index 0697b3d..321a5cf 100644 --- a/webapp/src/test/java/org/apache/falcon/resource/TestContext.java +++ b/webapp/src/test/java/org/apache/falcon/resource/TestContext.java @@ -87,17 +87,10 @@ import java.util.regex.Pattern; /** * Base test class for CLI, Entity and Process Instances. */ -public class TestContext { - public static final String FEED_TEMPLATE1 = "/feed-template1.xml"; - public static final String FEED_TEMPLATE2 = "/feed-template2.xml"; - public static final String FEED_TEMPLATE3 = "/feed-template3.xml"; +public class TestContext extends AbstractTestContext { public static final String DATASOURCE_TEMPLATE = "/datasource-template.xml"; - public static final String CLUSTER_TEMPLATE = "/cluster-template.xml"; - - public static final String SAMPLE_PROCESS_XML = "/process-version-0.xml"; - public static final String PROCESS_TEMPLATE = "/process-template.xml"; public static final String PIG_PROCESS_TEMPLATE = "/pig-process-template.xml"; public static final String BASE_URL = "https://localhost:41443/falcon-webapp";; @@ -477,14 +470,6 @@ public class TestContext { return tmpFile.getAbsolutePath(); } - public static File getTempFile() throws IOException { - return getTempFile("test", ".xml"); - } - - public static File getTempFile(String prefix, String suffix) throws IOException { - return getTempFile("target", prefix, suffix); - } - @SuppressWarnings("ResultOfMethodCallIgnored") public static File getTempFile(String path, String prefix, String suffix) throws IOException { File f = new File(path); @@ -523,18 +508,6 @@ public class TestContext { prepare(TestContext.CLUSTER_TEMPLATE, true); } - private static void mkdir(FileSystem fs, Path path) throws Exception { - if (!fs.exists(path) && !fs.mkdirs(path)) { - throw new Exception("mkdir failed for " + path); - } - } - - private static void mkdir(FileSystem fs, Path path, FsPermission perm) throws Exception { - if (!fs.exists(path) && !fs.mkdirs(path, perm)) { - throw new Exception("mkdir failed for " + path); - } - } - public static void prepare(String clusterTemplate, boolean disableLineage) throws Exception { // setup a logged in user CurrentUser.authenticate(REMOTE_USER); http://git-wip-us.apache.org/repos/asf/falcon/blob/94a70585/webapp/src/test/java/org/apache/falcon/resource/UnitTestContext.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/falcon/resource/UnitTestContext.java b/webapp/src/test/java/org/apache/falcon/resource/UnitTestContext.java index 1d3167b..2eb2e69 100644 --- a/webapp/src/test/java/org/apache/falcon/resource/UnitTestContext.java +++ b/webapp/src/test/java/org/apache/falcon/resource/UnitTestContext.java @@ -28,20 +28,16 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; -import java.io.File; import java.io.IOException; import java.util.Date; import java.util.HashMap; import java.util.Map; /** - * Test Utility. + * Context used to run unit tests using Falcon Unit. */ -public class UnitTestContext { +public class UnitTestContext extends AbstractTestContext { - public static final String FEED_TEMPLATE1 = "/feed-template1.xml"; - public static final String FEED_TEMPLATE2 = "/feed-template2.xml"; - public static final String PROCESS_TEMPLATE = "/process-template.xml"; public static final String SAMPLE_PROCESS_XML = "/process-version-0.xml"; protected String colo; @@ -71,20 +67,12 @@ public class UnitTestContext { return clusterName; } - public static FalconUnitClient getClient() { - return client; - } - - private static void mkdir(FileSystem fileSystem, Path path) throws Exception { - if (!fileSystem.exists(path) && !fileSystem.mkdirs(path)) { - throw new Exception("mkdir failed for " + path); - } + public String getInputFeedName() { + return inputFeedName; } - private static void mkdir(FileSystem fileSystem, Path path, FsPermission perm) throws Exception { - if (!fileSystem.exists(path) && !fileSystem.mkdirs(path, perm)) { - throw new Exception("mkdir failed for " + path); - } + public static FalconUnitClient getClient() { + return client; } protected void prepare(String workflow) throws Exception { @@ -108,24 +96,6 @@ public class UnitTestContext { prepare("sleepWorkflow.xml"); } - public static File getTempFile() throws IOException { - return getTempFile("test", ".xml"); - } - - public static File getTempFile(String prefix, String suffix) throws IOException { - return getTempFile("target", prefix, suffix); - } - - @SuppressWarnings("ResultOfMethodCallIgnored") - public static File getTempFile(String path, String prefix, String suffix) throws IOException { - File f = new File(path); - if (!f.exists()) { - f.mkdirs(); - } - - return File.createTempFile(prefix, suffix, f); - } - public Map<String, String> getUniqueOverlay() throws FalconException { Map<String, String> uniqueOverlay = new HashMap<String, String>(); long time = System.currentTimeMillis(); @@ -148,6 +118,7 @@ public class UnitTestContext { uniqueOverlay.put("user", System.getProperty("user.name")); uniqueOverlay.put("workflow.path", "/falcon/test/workflow"); uniqueOverlay.put("workflow.lib.path", "/falcon/test/workflow/lib"); + overlay = uniqueOverlay; return uniqueOverlay; } }
