Repository: falcon Updated Branches: refs/heads/master fd8614538 -> 03c86eb69
FALCON-964 add test in falcon regression to test loading of jar present in user lib. Contributed by Pragya M Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/03c86eb6 Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/03c86eb6 Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/03c86eb6 Branch: refs/heads/master Commit: 03c86eb696c22a96408fa69d0a16026057ac1d14 Parents: fd86145 Author: samarthg <[email protected]> Authored: Thu Mar 5 04:58:52 2015 +0000 Committer: samarthg <[email protected]> Committed: Thu Mar 5 04:58:52 2015 +0000 ---------------------------------------------------------------------- falcon-regression/CHANGES.txt | 3 + .../core/enumsAndConstants/MerlinConstants.java | 3 + .../falcon/regression/core/util/OSUtil.java | 4 + .../regression/ProcessLibPathLoadTest.java | 164 +++++++++++++++++++ .../src/test/resources/oozieLib/workflow.xml | 61 +++++++ 5 files changed, 235 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/03c86eb6/falcon-regression/CHANGES.txt ---------------------------------------------------------------------- diff --git a/falcon-regression/CHANGES.txt b/falcon-regression/CHANGES.txt index 1813952..09b535d 100644 --- a/falcon-regression/CHANGES.txt +++ b/falcon-regression/CHANGES.txt @@ -6,6 +6,9 @@ Trunk (Unreleased) NEW FEATURES + FALCON-964 add test in falcon regression to test loading of jar present in user lib(Pragya M + via Samarth Gupta) + FALCON-974 add test in falcon-regression for log mover feature(Pragya via Samarth Gupta) FALCON-843 add test to support current & last week el expression(Pragya M via Samarth G) http://git-wip-us.apache.org/repos/asf/falcon/blob/03c86eb6/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/enumsAndConstants/MerlinConstants.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/enumsAndConstants/MerlinConstants.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/enumsAndConstants/MerlinConstants.java index 1c517b4..0323fbc 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/enumsAndConstants/MerlinConstants.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/enumsAndConstants/MerlinConstants.java @@ -45,6 +45,9 @@ public final class MerlinConstants { public static final String WORKING_LOCATION = Config.getProperty("merlin.working.location", "/tmp/falcon-regression-working"); + public static final String OOZIE_EXAMPLE_LIB = Config.getProperty("merlin.oozie_example_lib", + "https://repo1.maven.org/maven2/org/apache/oozie/oozie-examples/4.1.0/oozie-examples-4.1.0.jar";); + /** the user that is going to run tests. */ public static final String CURRENT_USER_NAME = System.getProperty("user.name"); /** keytab of current user. */ http://git-wip-us.apache.org/repos/asf/falcon/blob/03c86eb6/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/OSUtil.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/OSUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/OSUtil.java index dd5e194..cc9e7ea 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/OSUtil.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/OSUtil.java @@ -47,6 +47,10 @@ public final class OSUtil { public static final String OOZIE_COMBINED_ACTIONS = String.format(RESOURCES + "combinedWorkflow%s", SEPARATOR); + public static final String OOZIE_LIB_FOLDER = + String.format(RESOURCES + "oozieLib%s", SEPARATOR); + + public static String getPath(String... pathParts) { return StringUtils.join(pathParts, OSUtil.SEPARATOR); } http://git-wip-us.apache.org/repos/asf/falcon/blob/03c86eb6/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathLoadTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathLoadTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathLoadTest.java new file mode 100644 index 0000000..74903e1 --- /dev/null +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathLoadTest.java @@ -0,0 +1,164 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.falcon.regression; + +import org.apache.falcon.entity.v0.EntityType; +import org.apache.falcon.regression.core.bundle.Bundle; +import org.apache.falcon.entity.v0.Frequency.TimeUnit; +import org.apache.falcon.regression.core.enumsAndConstants.MerlinConstants; +import org.apache.falcon.regression.core.helpers.ColoHelper; +import org.apache.falcon.regression.core.util.*; +import org.apache.falcon.regression.testHelper.BaseTestClass; +import org.apache.hadoop.fs.FileSystem; +import org.apache.log4j.Logger; +import org.apache.oozie.client.Job.Status; +import org.testng.Assert; +import org.testng.annotations.*; + +import java.io.*; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.List; +import java.util.Map; + +/** + * Tests with process lib folder with workflow.xml. + */ +@Test(groups = "embedded") +public class ProcessLibPathLoadTest extends BaseTestClass { + + + private ColoHelper cluster = servers.get(0); + private FileSystem clusterFS = serverFS.get(0); + private String testDir = cleanAndGetTestDir(); + private String aggregateWorkflowDir = testDir + "/aggregator"; + private static final Logger LOGGER = Logger.getLogger(ProcessLibPathLoadTest.class); + + private String oozieLib = MerlinConstants.OOZIE_EXAMPLE_LIB; + private String oozieLibName = oozieLib.substring(oozieLib.lastIndexOf('/') + 1); + private String filename = OSUtil.OOZIE_LIB_FOLDER + "lib/" + oozieLibName; + private String processName; + private String process; + + @BeforeClass(alwaysRun = true) + public void createTestData() throws Exception { + LOGGER.info("in @BeforeClass"); + saveUrlToFile(oozieLib); + } + + @BeforeMethod(alwaysRun = true) + public void setup() throws Exception { + uploadDirToClusters(aggregateWorkflowDir, OSUtil.OOZIE_LIB_FOLDER); + bundles[0] = BundleUtil.readELBundle(); + bundles[0] = new Bundle(bundles[0], cluster); + bundles[0].generateUniqueBundle(this); + bundles[0].setInputFeedDataPath(testDir + MINUTE_DATE_PATTERN); + bundles[0].setProcessValidity("2015-01-02T01:00Z", "2015-01-02T01:04Z"); + bundles[0].setProcessPeriodicity(5, TimeUnit.minutes); + bundles[0].setOutputFeedPeriodicity(5, TimeUnit.minutes); + bundles[0].setOutputFeedLocationData(testDir + "/output-data" + MINUTE_DATE_PATTERN); + bundles[0].setProcessConcurrency(1); + bundles[0].setProcessLibPath(aggregateWorkflowDir + "/lib"); + process = bundles[0].getProcessData(); + processName = Util.readEntityName(process); + } + + @AfterMethod(alwaysRun = true) + public void tearDown() { + removeTestClassEntities(); + } + + @AfterClass(alwaysRun = true) + public void deleteJar() { + File file = new File(filename); + Assert.assertEquals(file.delete(), true, filename + " is not present."); + } + + /** + * Test which test a process with jar in lib location. + * Schedule a process, it should succeed. + * + * @throws Exception + */ + @Test + public void setRightJarInWorkflowLib() throws Exception { + bundles[0].setProcessWorkflow(aggregateWorkflowDir); + bundles[0].submitFeedsScheduleProcess(prism); + InstanceUtil.waitTillInstancesAreCreated(cluster, process, 0); + OozieUtil.createMissingDependencies(cluster, EntityType.PROCESS, processName, 0); + InstanceUtil.waitForBundleToReachState(cluster, processName, Status.SUCCEEDED); + } + + /** + * Test which test a process with no jar in lib location. + * Schedule a process, it should get killed. + * + * @throws Exception + */ + @Test + public void setNoJarInWorkflowLibLocaltion() throws Exception { + HadoopUtil.deleteDirIfExists(aggregateWorkflowDir + "/lib/" + oozieLibName, clusterFS); + bundles[0].setProcessWorkflow(aggregateWorkflowDir); + bundles[0].submitFeedsScheduleProcess(prism); + InstanceUtil.waitTillInstancesAreCreated(cluster, process, 0); + OozieUtil.createMissingDependencies(cluster, EntityType.PROCESS, processName, 0); + InstanceUtil.waitForBundleToReachState(cluster, processName, Status.KILLED); + } + + /** + * Function to download jar at remote public location. + * @param urlString public location from where jar is to be downloaded + * filename is the location where the jar is to be saved + * @throws Exception + */ + private void saveUrlToFile(String urlString) + throws IOException { + + URL url = new URL(urlString); + String link; + HttpURLConnection http = (HttpURLConnection) url.openConnection(); + Map<String, List<String>> header = http.getHeaderFields(); + while (isRedirected(header)) { + link = header.get("Location").get(0); + url = new URL(link); + http = (HttpURLConnection) url.openConnection(); + header = http.getHeaderFields(); + } + + InputStream input = http.getInputStream(); + byte[] buffer = new byte[4096]; + int n = -1; + OutputStream output = new FileOutputStream(new File(filename)); + while ((n = input.read(buffer)) != -1) { + output.write(buffer, 0, n); + } + output.close(); + + } + + private static boolean isRedirected(Map<String, List<String>> header) { + for (String hv : header.get(null)) { + if (hv.contains(" 301 ") || hv.contains(" 302 ")) { + return true; + } + } + return false; + } + +} http://git-wip-us.apache.org/repos/asf/falcon/blob/03c86eb6/falcon-regression/merlin/src/test/resources/oozieLib/workflow.xml ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/resources/oozieLib/workflow.xml b/falcon-regression/merlin/src/test/resources/oozieLib/workflow.xml new file mode 100644 index 0000000..4a80109 --- /dev/null +++ b/falcon-regression/merlin/src/test/resources/oozieLib/workflow.xml @@ -0,0 +1,61 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<workflow-app xmlns="uri:oozie:workflow:0.2" name="aggregator-wf"> + <start to="aggregator"/> + <action name="aggregator"> + <map-reduce> + <job-tracker>${jobTracker}</job-tracker> + <name-node>${nameNode}</name-node> + <prepare> + <delete path="${outputData}"/> + </prepare> + <configuration> + <property> + <name>mapred.job.queue.name</name> + <value>${queueName}</value> + </property> + <property> + <name>mapred.mapper.class</name> + <value>org.apache.oozie.example.SampleMapper</value> + </property> + <property> + <name>mapred.reducer.class</name> + <value>org.apache.oozie.example.SampleReducer</value> + </property> + <property> + <name>mapred.map.tasks</name> + <value>1</value> + </property> + <property> + <name>mapred.input.dir</name> + <value>${inputData}</value> + </property> + <property> + <name>mapred.output.dir</name> + <value>${outputData}</value> + </property> + </configuration> + </map-reduce> + <ok to="end"/> + <error to="fail"/> + </action> + <kill name="fail"> + <message>Map/Reduce failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message> + </kill> + <end name="end"/> +</workflow-app>
