Repository: incubator-falcon Updated Branches: refs/heads/master 73c9b9f54 -> 4133c9633
FALCON-908 Remove jars that are not needed. Contributed by Paul Isaychuk Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/4133c963 Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/4133c963 Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/4133c963 Branch: refs/heads/master Commit: 4133c9633baf709a234c493cb973ac30d3bcb14c Parents: 73c9b9f Author: Raghav Kumar Gautam <rag...@apache.org> Authored: Wed Dec 10 13:24:45 2014 -0800 Committer: Raghav Kumar Gautam <rag...@apache.org> Committed: Wed Dec 10 13:24:45 2014 -0800 ---------------------------------------------------------------------- falcon-regression/CHANGES.txt | 2 + .../falcon/regression/core/bundle/Bundle.java | 11 ++-- .../helpers/entity/AbstractEntityHelper.java | 32 +++++----- .../falcon/regression/ProcessLibPathTest.java | 10 +-- .../resources/ELbundle/workflow/workflow.xml | 61 ------------------- .../src/test/resources/IvoryClient/IvoryCLI.jar | Bin 4386324 -> 0 bytes .../src/test/resources/ivory-oozie-lib-0.1.jar | Bin 8530 -> 0 bytes .../oozie/lib/oozie-examples-3.1.5.jar | Bin 15826 -> 0 bytes .../src/test/resources/oozie/workflow.xml | 4 +- .../lib/oozie-examples-3.1.5.jar | Bin 15826 -> 0 bytes 10 files changed, 28 insertions(+), 92 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4133c963/falcon-regression/CHANGES.txt ---------------------------------------------------------------------- diff --git a/falcon-regression/CHANGES.txt b/falcon-regression/CHANGES.txt index 8ae1c7a..f7b9781 100644 --- a/falcon-regression/CHANGES.txt +++ b/falcon-regression/CHANGES.txt @@ -36,6 +36,8 @@ Trunk (Unreleased) via Samarth Gupta) IMPROVEMENTS + FALCON-908 Remove jars that are not needed (Paul Isaychuk via Raghav Kumar Gautam) + FALCON-939 Fixing few typos and removing unused stuff (Paul Isaychuk via Ruslan Ostafiychuk) FALCON-930 Delete old project name (ivory) from code and xmls (Ruslan Ostafiychuk via Raghav http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4133c963/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java index 4c913f5..91abe89 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java @@ -572,10 +572,11 @@ public class Bundle { } - public void verifyDependencyListing(ColoHelper coloHelper) { + public void verifyDependencyListing(ColoHelper coloHelper) + throws InterruptedException, IOException, AuthenticationException, URISyntaxException { //display dependencies of process: String dependencies = coloHelper.getProcessHelper().getDependencies( - Util.readEntityName(getProcessData())); + Util.readEntityName(getProcessData())).getEntityList().toString(); //verify presence for (String cluster : clusters) { @@ -585,15 +586,13 @@ public class Bundle { Assert.assertTrue(dependencies.contains("(feed) " + Util.readEntityName(feed))); for (String cluster : clusters) { Assert.assertTrue(coloHelper.getFeedHelper().getDependencies( - Util.readEntityName(feed)) + Util.readEntityName(feed)).getEntityList().toString() .contains("(cluster) " + Util.readEntityName(cluster))); } Assert.assertFalse(coloHelper.getFeedHelper().getDependencies( - Util.readEntityName(feed)) + Util.readEntityName(feed)).getEntityList().toString() .contains("(process)" + Util.readEntityName(getProcessData()))); } - - } public void addProcessInput(String feed, String feedName) { http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4133c963/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java index 797554b..b365a4c 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java @@ -29,7 +29,6 @@ import org.apache.falcon.regression.core.util.ExecUtil; import org.apache.falcon.regression.core.util.FileUtil; import org.apache.falcon.regression.core.util.HCatUtil; import org.apache.falcon.regression.core.util.InstanceUtil; -import org.apache.falcon.regression.core.util.OSUtil; import org.apache.falcon.regression.core.util.OozieUtil; import org.apache.falcon.regression.core.util.Util; import org.apache.falcon.regression.core.util.Util.URLS; @@ -56,10 +55,6 @@ public abstract class AbstractEntityHelper { private static final Logger LOGGER = Logger.getLogger(AbstractEntityHelper.class); - protected static final String CLIENT_LOCATION = OSUtil.RESOURCES - + OSUtil.getPath("IvoryClient", "IvoryCLI.jar"); - protected static final String BASE_COMMAND = "java -jar " + CLIENT_LOCATION; - private static boolean setAuthenticate() { String value = Config.getProperty("isAuthenticationSet"); value = (null == value) ? "true" : value; @@ -495,17 +490,6 @@ public abstract class AbstractEntityHelper { .createAndSendRequestProcessInstance(url, params, allColo, null); } - public String list() { - return ExecUtil.executeCommandGetOutput( - BASE_COMMAND + " entity -list -url " + this.hostname + " -type " + getEntityType()); - } - - public String getDependencies(String entityName) { - return ExecUtil.executeCommandGetOutput( - BASE_COMMAND + " entity -dependency -url " + this.hostname + " -type " - + getEntityType() + " -name " + entityName); - } - public List<String> getArchiveInfo() throws IOException, JSchException { return Util.getStoreInfo(this, "/archive/" + getEntityType().toUpperCase()); } @@ -597,4 +581,20 @@ public abstract class AbstractEntityHelper { } return (InstancesResult) InstanceUtil.sendRequestProcessInstance(url, user); } + + /** + * Get list of all dependencies of a given entity. + * @param entityName entity name + * @return response + * @throws URISyntaxException + * @throws AuthenticationException + * @throws InterruptedException + * @throws IOException + */ + public ServiceResponse getDependencies(String entityName) + throws URISyntaxException, AuthenticationException, InterruptedException, IOException { + String url = createUrl(this.hostname + URLS.DEPENDENCIES.getValue(), getEntityType(), + entityName + colo); + return Util.sendRequest(url, "get", null, null); + } } http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4133c963/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathTest.java index 6b1dec3..f335f04 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathTest.java @@ -60,9 +60,6 @@ public class ProcessLibPathTest extends BaseTestClass { @BeforeClass(alwaysRun = true) public void createTestData() throws Exception { LOGGER.info("in @BeforeClass"); - - //common lib for both test cases - HadoopUtil.uploadDir(clusterFS, testLibDir, OSUtil.RESOURCES_OOZIE + "lib"); Bundle b = BundleUtil.readELBundle(); b.generateUniqueBundle(); b = new Bundle(b, cluster); @@ -98,7 +95,7 @@ public class ProcessLibPathTest extends BaseTestClass { } /** - * Test which test a process with no lib folder in workflow location. + * Tests a process with no lib folder in workflow location. * * @throws Exception */ @@ -106,7 +103,6 @@ public class ProcessLibPathTest extends BaseTestClass { public void setDifferentLibPathWithNoLibFolderInWorkflowfLocaltion() throws Exception { String workflowDir = testLibDir + "/aggregatorLib1/"; HadoopUtil.uploadDir(clusterFS, workflowDir, OSUtil.RESOURCES_OOZIE); - HadoopUtil.deleteDirIfExists(workflowDir + "/lib", clusterFS); bundles[0].setProcessWorkflow(workflowDir); LOGGER.info("processData: " + Util.prettyPrintXml(process)); bundles[0].submitFeedsScheduleProcess(prism); @@ -125,8 +121,8 @@ public class ProcessLibPathTest extends BaseTestClass { String workflowDir = testLibDir + "/aggregatorLib2/"; HadoopUtil.uploadDir(clusterFS, workflowDir, OSUtil.RESOURCES_OOZIE); HadoopUtil.recreateDir(clusterFS, workflowDir + "/lib"); - HadoopUtil.copyDataToFolder(clusterFS, workflowDir + "/lib", - OSUtil.RESOURCES + "ivory-oozie-lib-0.1.jar"); + HadoopUtil.copyDataToFolder(clusterFS, workflowDir + "/lib/invalid.jar", + OSUtil.RESOURCES + "feed-s4Replication.xml"); bundles[0].setProcessWorkflow(workflowDir); LOGGER.info("processData: " + Util.prettyPrintXml(process)); bundles[0].submitFeedsScheduleProcess(prism); http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4133c963/falcon-regression/merlin/src/test/resources/ELbundle/workflow/workflow.xml ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/resources/ELbundle/workflow/workflow.xml b/falcon-regression/merlin/src/test/resources/ELbundle/workflow/workflow.xml deleted file mode 100644 index 4a80109..0000000 --- a/falcon-regression/merlin/src/test/resources/ELbundle/workflow/workflow.xml +++ /dev/null @@ -1,61 +0,0 @@ -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<workflow-app xmlns="uri:oozie:workflow:0.2" name="aggregator-wf"> - <start to="aggregator"/> - <action name="aggregator"> - <map-reduce> - <job-tracker>${jobTracker}</job-tracker> - <name-node>${nameNode}</name-node> - <prepare> - <delete path="${outputData}"/> - </prepare> - <configuration> - <property> - <name>mapred.job.queue.name</name> - <value>${queueName}</value> - </property> - <property> - <name>mapred.mapper.class</name> - <value>org.apache.oozie.example.SampleMapper</value> - </property> - <property> - <name>mapred.reducer.class</name> - <value>org.apache.oozie.example.SampleReducer</value> - </property> - <property> - <name>mapred.map.tasks</name> - <value>1</value> - </property> - <property> - <name>mapred.input.dir</name> - <value>${inputData}</value> - </property> - <property> - <name>mapred.output.dir</name> - <value>${outputData}</value> - </property> - </configuration> - </map-reduce> - <ok to="end"/> - <error to="fail"/> - </action> - <kill name="fail"> - <message>Map/Reduce failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message> - </kill> - <end name="end"/> -</workflow-app> http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4133c963/falcon-regression/merlin/src/test/resources/IvoryClient/IvoryCLI.jar ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/resources/IvoryClient/IvoryCLI.jar b/falcon-regression/merlin/src/test/resources/IvoryClient/IvoryCLI.jar deleted file mode 100644 index 319bf5a..0000000 Binary files a/falcon-regression/merlin/src/test/resources/IvoryClient/IvoryCLI.jar and /dev/null differ http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4133c963/falcon-regression/merlin/src/test/resources/ivory-oozie-lib-0.1.jar ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/resources/ivory-oozie-lib-0.1.jar b/falcon-regression/merlin/src/test/resources/ivory-oozie-lib-0.1.jar deleted file mode 100755 index 992feab..0000000 Binary files a/falcon-regression/merlin/src/test/resources/ivory-oozie-lib-0.1.jar and /dev/null differ http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4133c963/falcon-regression/merlin/src/test/resources/oozie/lib/oozie-examples-3.1.5.jar ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/resources/oozie/lib/oozie-examples-3.1.5.jar b/falcon-regression/merlin/src/test/resources/oozie/lib/oozie-examples-3.1.5.jar deleted file mode 100755 index 380c6f3..0000000 Binary files a/falcon-regression/merlin/src/test/resources/oozie/lib/oozie-examples-3.1.5.jar and /dev/null differ http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4133c963/falcon-regression/merlin/src/test/resources/oozie/workflow.xml ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/resources/oozie/workflow.xml b/falcon-regression/merlin/src/test/resources/oozie/workflow.xml index 4a80109..8c9cdaf 100644 --- a/falcon-regression/merlin/src/test/resources/oozie/workflow.xml +++ b/falcon-regression/merlin/src/test/resources/oozie/workflow.xml @@ -31,11 +31,11 @@ </property> <property> <name>mapred.mapper.class</name> - <value>org.apache.oozie.example.SampleMapper</value> + <value>org.apache.hadoop.mapred.lib.IdentityMapper</value> </property> <property> <name>mapred.reducer.class</name> - <value>org.apache.oozie.example.SampleReducer</value> + <value>org.apache.hadoop.mapred.lib.IdentityReducer</value> </property> <property> <name>mapred.map.tasks</name> http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4133c963/falcon-regression/merlin/src/test/resources/workflows/aggregatorNoOutput/lib/oozie-examples-3.1.5.jar ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/resources/workflows/aggregatorNoOutput/lib/oozie-examples-3.1.5.jar b/falcon-regression/merlin/src/test/resources/workflows/aggregatorNoOutput/lib/oozie-examples-3.1.5.jar deleted file mode 100644 index 380c6f3..0000000 Binary files a/falcon-regression/merlin/src/test/resources/workflows/aggregatorNoOutput/lib/oozie-examples-3.1.5.jar and /dev/null differ
