Repository: falcon Updated Branches: refs/heads/master b5c0b6139 -> 4a76b11be
FALCON-1766 Add CLI metrics check for HiveDR, HDFS and feed replication. Contributed by Paul Isaychuk Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/4a76b11b Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/4a76b11b Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/4a76b11b Branch: refs/heads/master Commit: 4a76b11be2a6f7831929831d91ecc5ac46e25260 Parents: b5c0b61 Author: Paul Isaychuk <[email protected]> Authored: Thu Jan 21 17:03:55 2016 +0200 Committer: Paul Isaychuk <[email protected]> Committed: Tue Feb 2 15:07:18 2016 +0200 ---------------------------------------------------------------------- falcon-regression/CHANGES.txt | 2 + .../core/helpers/FalconClientBuilder.java | 13 ++++ .../helpers/entity/AbstractEntityHelper.java | 12 +++ .../falcon/regression/core/util/AssertUtil.java | 42 +++++++++- .../falcon/regression/FeedReplicationTest.java | 30 ++++--- .../regression/hive/dr/HdfsRecipeTest.java | 5 ++ .../falcon/regression/hive/dr/HiveDRTest.java | 5 ++ .../HdfsRecipe/hdfs-replication-template.xml | 44 +++++++++++ .../HdfsRecipe/hdfs-replication-workflow.xml | 82 ++++++++++++++++++++ .../HdfsRecipe/hdfs-replication.properties | 79 +++++++++++++++++++ .../hive-disaster-recovery-template.xml | 44 ----------- .../hive-disaster-recovery-workflow.xml | 82 -------------------- .../hive-disaster-recovery.properties | 79 ------------------- .../hive-disaster-recovery-workflow.xml | 2 + 14 files changed, 303 insertions(+), 218 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/4a76b11b/falcon-regression/CHANGES.txt ---------------------------------------------------------------------- diff --git a/falcon-regression/CHANGES.txt b/falcon-regression/CHANGES.txt index e6664f3..ccd9ceb 100644 --- a/falcon-regression/CHANGES.txt +++ b/falcon-regression/CHANGES.txt @@ -5,6 +5,8 @@ Trunk (Unreleased) INCOMPATIBLE CHANGES NEW FEATURES + FALCON-1766 Add CLI metrics check for HiveDR, HDFS and feed replication (Paul Isaychuk) + FALCON-1777 Add regression for HDFS replication (recipe) (Paul Isaychuk) FALCON-1699 Test fixes for RetentionTest, LineageApiTest, TouchAPIPrismAndServerTest, FeedReplicationTest and few fortifications(Paul Isaychuk via Pragya Mittal) http://git-wip-us.apache.org/repos/asf/falcon/blob/4a76b11b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/FalconClientBuilder.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/FalconClientBuilder.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/FalconClientBuilder.java index 2e6fca9..fcbb7f9 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/FalconClientBuilder.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/FalconClientBuilder.java @@ -139,6 +139,19 @@ public final class FalconClientBuilder implements Builder<CommandLine> { } /** + * Create get metrics command. + * @param entityType type of the entity + * @param entityName name of the entity to be deleted + * @return this + */ + public FalconClientBuilder getMetricsCommand(String entityType, String entityName) { + addArg("metadata").addArg("-list"); + addArg("-type").addArg("replication_metrics"); + addArg("-" + entityType).addArg(entityName); + return this; + } + + /** * Create submit command. * @param entityType type of the entity * @param fileName file containing the entity to be submitted http://git-wip-us.apache.org/repos/asf/falcon/blob/4a76b11b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java index e406cae..29c97b2 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java @@ -593,6 +593,18 @@ public abstract class AbstractEntityHelper { } /** + * Get CLI metrics for recipe based process or feed replication. + * @param entityName + * @return + */ + public ExecResult getCLIMetrics(String entityName) { + LOGGER.info("Getting CLI metrics for " + getEntityType()+ " " + entityName); + final CommandLine commandLine = FalconClientBuilder.getBuilder() + .getMetricsCommand(getEntityType(), entityName).build(); + return ExecUtil.executeCommand(commandLine); + } + + /** * Delete an entity through falcon client. * @param entityStr string of the entity to be submitted * @throws IOException http://git-wip-us.apache.org/repos/asf/falcon/blob/4a76b11b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/AssertUtil.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/AssertUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/AssertUtil.java index 9d3b802..0af0c1e 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/AssertUtil.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/AssertUtil.java @@ -33,6 +33,8 @@ import org.apache.log4j.Logger; import org.apache.oozie.client.Job; import org.apache.oozie.client.OozieClient; import org.apache.oozie.client.OozieClientException; +import org.json.JSONArray; +import org.json.JSONObject; import org.testng.Assert; import javax.xml.bind.JAXBException; @@ -52,6 +54,38 @@ public final class AssertUtil { private static final Logger LOGGER = Logger.getLogger(AssertUtil.class); /** + * Asserts correctness of CLI metrics for recipe based process or feed replication. + * @param execResult CLI metrics exec result to be checked + * @param entityName name of recipe process or replication feed + * @param instanceNum expected number of process/feed instances in metrics output + * @param withData is data expected to be replicated + * @throws Exception + */ + public static void assertCLIMetrics(ExecResult execResult, String entityName, int instanceNum, boolean withData) + throws Exception { + String output = execResult.getOutput(); + Assert.assertTrue(StringUtils.isNotBlank(output), "Exec result output is blank."); + JSONObject jsonObject = new JSONObject(output); + int totalSize = jsonObject.getInt("totalSize"); + Assert.assertEquals(totalSize, instanceNum); + JSONArray array = jsonObject.getJSONArray("results"); + for (int i = 0; i < array.length(); i++) { + String name = array.getJSONObject(i).getString("name"); + Assert.assertTrue(name.contains(entityName)); + int timeTaken = array.getJSONObject(i).getInt("TIMETAKEN"); + Assert.assertTrue(timeTaken > 0, "TIMETAKEN metric should be greater then zero."); + int bytescopied = array.getJSONObject(i).getInt("BYTESCOPIED"); + Assert.assertTrue(bytescopied >= 0, "BYTESCOPIED metric should be greater or equal to zero."); + int copy = array.getJSONObject(i).getInt("COPY"); + if (withData) { + Assert.assertTrue(copy > 0, "COPY metric should be greater then zero."); + } else { + Assert.assertEquals(copy, 0, "COPY metric should be equal to zero as data was absent."); + } + } + } + + /** * Checks that any path in list doesn't contains a string. * * @param paths list of paths @@ -96,7 +130,7 @@ public final class AssertUtil { LOGGER.info("elements = " + elements); } Assert.assertEquals(elements.size(), expectedSize, - "Size of expected and actual list don't match."); + "Size of expected and actual list don't match."); } /** @@ -218,7 +252,7 @@ public final class AssertUtil { Assert.assertFalse(execResult.hasSuceeded(), "Unexpectedly succeeded execResult: " + execResult); Assert.assertTrue((execResult.getError() + execResult.getOutput()).contains(expectedMessage), - "Expected error: " + expectedMessage + " in execResult: " + execResult); + "Expected error: " + expectedMessage + " in execResult: " + execResult); } /** @@ -258,7 +292,7 @@ public final class AssertUtil { */ public static void assertFailed(ServiceResponse response) throws JAXBException { Assert.assertNotEquals(response.getMessage(), "null", - "response message should not be null"); + "response message should not be null"); Assert.assertEquals(Util.parseResponse(response).getStatus(), APIResult.Status.FAILED); Assert.assertEquals(response.getCode(), 400); @@ -271,7 +305,7 @@ public final class AssertUtil { */ public static void assertFailed(APIResult response) { Assert.assertNotEquals(response.getMessage(), "null", - "response message should not be null"); + "response message should not be null"); Assert.assertEquals(response.getStatus(), APIResult.Status.FAILED, "Status should be FAILED. Message: " + response.getMessage()); } http://git-wip-us.apache.org/repos/asf/falcon/blob/4a76b11b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java index 6728edf..a936aa1 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java @@ -24,6 +24,7 @@ import org.apache.falcon.entity.v0.feed.ClusterType; import org.apache.falcon.regression.Entities.FeedMerlin; import org.apache.falcon.regression.core.bundle.Bundle; import org.apache.falcon.regression.core.helpers.ColoHelper; +import org.apache.falcon.regression.core.supportClasses.ExecResult; import org.apache.falcon.regression.core.util.AssertUtil; import org.apache.falcon.regression.core.util.BundleUtil; import org.apache.falcon.regression.core.util.HadoopUtil; @@ -36,11 +37,9 @@ import org.apache.falcon.regression.testHelper.BaseTestClass; import org.apache.falcon.resource.InstancesResult; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.security.authentication.client.AuthenticationException; import org.apache.log4j.Logger; import org.apache.oozie.client.CoordinatorAction; import org.apache.oozie.client.OozieClient; -import org.apache.oozie.client.OozieClientException; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.joda.time.format.DateTimeFormat; @@ -53,7 +52,6 @@ import org.testng.annotations.Test; import javax.xml.bind.JAXBException; import java.io.IOException; -import java.net.URISyntaxException; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -133,6 +131,7 @@ public class FeedReplicationTest extends BaseTestClass { .withClusterType(ClusterType.TARGET) .withDataLocation(targetDataLocation) .build()); + feed.withProperty("job.counter", "true"); //submit and schedule feed LOGGER.info("Feed : " + Util.prettyPrintXml(feed.toString())); @@ -178,7 +177,10 @@ public class FeedReplicationTest extends BaseTestClass { Assert.assertEquals(HadoopUtil.getSuccessFolder(cluster2FS, toTarget, ""), true); AssertUtil.assertLogMoverPath(true, Util.readEntityName(feed.toString()), - cluster2FS, "feed", "Success logs are not present"); + cluster2FS, "feed", "Success logs are not present"); + + ExecResult execResult = cluster1.getFeedHelper().getCLIMetrics(feed.getName()); + AssertUtil.assertCLIMetrics(execResult, feed.getName(), 1, dataFlag); } /** @@ -222,6 +224,7 @@ public class FeedReplicationTest extends BaseTestClass { .withClusterType(ClusterType.TARGET) .withDataLocation(targetDataLocation) .build()); + feed.withProperty("job.counter", "true"); //submit and schedule feed LOGGER.info("Feed : " + Util.prettyPrintXml(feed.toString())); @@ -278,7 +281,10 @@ public class FeedReplicationTest extends BaseTestClass { Assert.assertEquals(HadoopUtil.getSuccessFolder(cluster3FS, toTarget, ""), true); AssertUtil.assertLogMoverPath(true, Util.readEntityName(feed.toString()), - cluster2FS, "feed", "Success logs are not present"); + cluster2FS, "feed", "Success logs are not present"); + + ExecResult execResult = cluster1.getFeedHelper().getCLIMetrics(feed.getName()); + AssertUtil.assertCLIMetrics(execResult, feed.getName(), 1, dataFlag); } /** @@ -322,6 +328,7 @@ public class FeedReplicationTest extends BaseTestClass { .withClusterType(ClusterType.TARGET) .withDataLocation(targetDataLocation) .build()); + feed.withProperty("job.counter", "true"); //submit and schedule feed LOGGER.info("Feed : " + Util.prettyPrintXml(feed.toString())); @@ -384,7 +391,10 @@ public class FeedReplicationTest extends BaseTestClass { Assert.assertEquals(HadoopUtil.getSuccessFolder(cluster2FS, toTarget, availabilityFlagName), true); AssertUtil.assertLogMoverPath(true, Util.readEntityName(feed.toString()), - cluster2FS, "feed", "Success logs are not present"); + cluster2FS, "feed", "Success logs are not present"); + + ExecResult execResult = cluster1.getFeedHelper().getCLIMetrics(feed.getName()); + AssertUtil.assertCLIMetrics(execResult, feed.getName(), 1, dataFlag); } /** @@ -392,9 +402,7 @@ public class FeedReplicationTest extends BaseTestClass { * Check that new DistCp options are allowed. */ @Test - public void testNewDistCpOptions() - throws URISyntaxException, AuthenticationException, InterruptedException, IOException, JAXBException, - OozieClientException { + public void testNewDistCpOptions() throws Exception { Bundle.submitCluster(bundles[0], bundles[1]); String startTime = TimeUtil.getTimeWrtSystemTime(0); String endTime = TimeUtil.addMinsToTime(startTime, 5); @@ -422,6 +430,7 @@ public class FeedReplicationTest extends BaseTestClass { .withClusterType(ClusterType.TARGET) .withDataLocation(targetDataLocation) .build()); + feed.withProperty("job.counter", "true"); //add custom properties to feed HashMap<String, String> propMap = new HashMap<>(); @@ -485,6 +494,9 @@ public class FeedReplicationTest extends BaseTestClass { List<Path> finalFiles = HadoopUtil.getAllFilesRecursivelyHDFS(cluster2FS, new Path(targetPath)); Assert.assertEquals(finalFiles.size(), 2, "Only replicated files should be present on target " + "because of 'removeDeletedFiles' distCp property."); + + ExecResult execResult = cluster1.getFeedHelper().getCLIMetrics(feed.getName()); + AssertUtil.assertCLIMetrics(execResult, feed.getName(), 1, true); } /** http://git-wip-us.apache.org/repos/asf/falcon/blob/4a76b11b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hive/dr/HdfsRecipeTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hive/dr/HdfsRecipeTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hive/dr/HdfsRecipeTest.java index 05b9cf4..4a2d913 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hive/dr/HdfsRecipeTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hive/dr/HdfsRecipeTest.java @@ -25,6 +25,7 @@ import org.apache.falcon.regression.Entities.ClusterMerlin; import org.apache.falcon.regression.Entities.RecipeMerlin; import org.apache.falcon.regression.core.bundle.Bundle; import org.apache.falcon.regression.core.helpers.ColoHelper; +import org.apache.falcon.regression.core.supportClasses.ExecResult; import org.apache.falcon.regression.core.util.AssertUtil; import org.apache.falcon.regression.core.util.BundleUtil; import org.apache.falcon.regression.core.util.HadoopUtil; @@ -111,6 +112,10 @@ public class HdfsRecipeTest extends BaseTestClass { .getAllFilesRecursivelyHDFS(clusterFS2, new Path(targetDataLocation)); AssertUtil.checkForListSizes(cluster1ReplicatedData, cluster2ReplicatedData); + + //particular check for https://issues.apache.org/jira/browse/FALCON-1643 + ExecResult execResult = cluster.getProcessHelper().getCLIMetrics(hdfsRecipe.getName()); + AssertUtil.assertCLIMetrics(execResult, hdfsRecipe.getName(), 1, true); } @AfterMethod(alwaysRun = true) http://git-wip-us.apache.org/repos/asf/falcon/blob/4a76b11b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hive/dr/HiveDRTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hive/dr/HiveDRTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hive/dr/HiveDRTest.java index 7660e04..4dab0db 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hive/dr/HiveDRTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hive/dr/HiveDRTest.java @@ -26,6 +26,8 @@ import org.apache.falcon.regression.Entities.RecipeMerlin; import org.apache.falcon.regression.core.bundle.Bundle; import org.apache.falcon.regression.core.enumsAndConstants.MerlinConstants; import org.apache.falcon.regression.core.helpers.ColoHelper; +import org.apache.falcon.regression.core.supportClasses.ExecResult; +import org.apache.falcon.regression.core.util.AssertUtil; import org.apache.falcon.regression.core.supportClasses.NotifyingAssert; import org.apache.falcon.regression.core.util.BundleUtil; import org.apache.falcon.regression.core.util.HiveAssert; @@ -164,6 +166,9 @@ public class HiveDRTest extends BaseTestClass { HiveAssert.assertTableEqual(cluster, clusterHC.getTable(DB_NAME, tblName), cluster2, clusterHC2.getTable(DB_NAME, tblName), new NotifyingAssert(true) ).assertAll(); + + ExecResult execResult = cluster.getProcessHelper().getCLIMetrics(recipeMerlin.getName()); + AssertUtil.assertCLIMetrics(execResult, recipeMerlin.getName(), 1, true); } @Test http://git-wip-us.apache.org/repos/asf/falcon/blob/4a76b11b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication-template.xml ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication-template.xml b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication-template.xml new file mode 100644 index 0000000..46546ce --- /dev/null +++ b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication-template.xml @@ -0,0 +1,44 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<process name="##falcon.recipe.job.name##" xmlns="uri:falcon:process:0.1"> + <clusters> + <!-- source --> + <cluster name="##falcon.recipe.cluster.name##"> + <validity end="##falcon.recipe.cluster.validity.end##" start="##falcon.recipe.cluster.validity.start##"/> + </cluster> + </clusters> + + <tags>_falcon_mirroring_type=HDFS</tags> + + <parallel>1</parallel> + <!-- Dir replication needs to run only once to catch up --> + <order>LAST_ONLY</order> + <frequency>##falcon.recipe.frequency##</frequency> + <timezone>UTC</timezone> + + <properties> + <property name="oozie.wf.subworkflow.classpath.inheritance" value="true"/> + </properties> + + <workflow name="##falcon.recipe.workflow.name##" engine="oozie" path="/apps/data-mirroring/workflows/hdfs-replication-workflow.xml" lib="##workflow.lib.path##"/> + <retry policy="##falcon.recipe.retry.policy##" delay="##falcon.recipe.retry.delay##" attempts="3"/> + <notification type="##falcon.recipe.notification.type##" to="##falcon.recipe.notification.receivers##"/> + <ACL/> +</process> http://git-wip-us.apache.org/repos/asf/falcon/blob/4a76b11b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication-workflow.xml ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication-workflow.xml b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication-workflow.xml new file mode 100644 index 0000000..aa820d0 --- /dev/null +++ b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication-workflow.xml @@ -0,0 +1,82 @@ +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> +<workflow-app xmlns='uri:oozie:workflow:0.3' name='falcon-dr-fs-workflow'> + <start to='dr-replication'/> + <!-- Replication action --> + <action name="dr-replication"> + <java> + <job-tracker>${jobTracker}</job-tracker> + <name-node>${nameNode}</name-node> + <configuration> + <property> <!-- hadoop 2 parameter --> + <name>oozie.launcher.mapreduce.job.user.classpath.first</name> + <value>true</value> + </property> + <property> + <name>mapred.job.queue.name</name> + <value>${queueName}</value> + </property> + <property> + <name>oozie.launcher.mapred.job.priority</name> + <value>${jobPriority}</value> + </property> + <property> + <name>oozie.use.system.libpath</name> + <value>true</value> + </property> + <property> + <name>oozie.action.sharelib.for.java</name> + <value>distcp</value> + </property> + <property> + <name>oozie.launcher.oozie.libpath</name> + <value>${wf:conf("falcon.libpath")}</value> + </property> + <property> + <name>oozie.launcher.mapreduce.job.hdfs-servers</name> + <value>${drSourceClusterFS},${drTargetClusterFS}</value> + </property> + </configuration> + <main-class>org.apache.falcon.replication.FeedReplicator</main-class> + <arg>-Dmapred.job.queue.name=${queueName}</arg> + <arg>-Dmapred.job.priority=${jobPriority}</arg> + <arg>-maxMaps</arg> + <arg>${distcpMaxMaps}</arg> + <arg>-mapBandwidth</arg> + <arg>${distcpMapBandwidth}</arg> + <arg>-sourcePaths</arg> + <arg>${drSourceDir}</arg> + <arg>-targetPath</arg> + <arg>${drTargetClusterFS}${drTargetDir}</arg> + <arg>-falconFeedStorageType</arg> + <arg>FILESYSTEM</arg> + <arg>-availabilityFlag</arg> + <arg>${availabilityFlag == 'NA' ? "NA" : availabilityFlag}</arg> + <arg>-counterLogDir</arg> + <arg>${logDir}/job-${nominalTime}/${srcClusterName == 'NA' ? '' : srcClusterName}</arg> + </java> + <ok to="end"/> + <error to="fail"/> + </action> + <kill name="fail"> + <message> + Workflow action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] + </message> + </kill> + <end name="end"/> +</workflow-app> http://git-wip-us.apache.org/repos/asf/falcon/blob/4a76b11b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication.properties ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication.properties b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication.properties new file mode 100644 index 0000000..6c715f3 --- /dev/null +++ b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication.properties @@ -0,0 +1,79 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +##### NOTE: This is a TEMPLATE file which can be copied and edited + +##### Recipe properties +##### Unique recipe job name +falcon.recipe.name=hdfs-replication-monthly + +##### Workflow properties +falcon.recipe.workflow.name=hdfs-dr-workflow +# Provide Wf absolute path. This can be HDFS or local FS path. If WF is on local FS it will be copied to HDFS +falcon.recipe.workflow.path=/apps/data-mirroring/workflows/hdfs-replication-workflow.xml + +##### Cluster properties + +# Cluster where job should run +falcon.recipe.cluster.name=primaryCluster +# Change the cluster hdfs write end point here. This is mandatory. +falcon.recipe.cluster.hdfs.writeEndPoint=hdfs://localhost:8020 +# Change the cluster validity start time here +falcon.recipe.cluster.validity.start=2015-03-13T00:00Z +# Change the cluster validity end time here +falcon.recipe.cluster.validity.end=2016-12-30T00:00Z + +##### Scheduling properties +# Change the recipe frequency here. Valid frequency type are minutes, hours, days, months +falcon.recipe.process.frequency=minutes(5) + +##### Retry policy properties + +falcon.recipe.retry.policy=periodic +falcon.recipe.retry.delay=minutes(30) +falcon.recipe.retry.attempts=3 +falcon.recipe.retry.onTimeout=false + +##### Tag properties - An optional list of comma separated tags, Key Value Pairs, separated by comma +##### Uncomment to add tags +#falcon.recipe.tags= + +##### ACL properties - Uncomment and change ACL if authorization is enabled + +#falcon.recipe.acl.owner=ambari-qa +#falcon.recipe.acl.group=users +#falcon.recipe.acl.permission=0x755 +#falcon.recipe.nn.principal=nn/[email protected] + +##### Custom Job properties + +# Specify multiple comma separated source directories +drSourceDir=/user/falcon_qa/dr/test/primaryCluster/input +drSourceClusterFS=hdfs://240.0.0.10:8020 +drTargetDir=/user/falcon_qa/dr/test/backupCluster/input +drTargetClusterFS=hdfs://240.0.0.11:8020 + +# Change it to specify the maximum number of mappers for DistCP +distcpMaxMaps=1 +# Change it to specify the bandwidth in MB for each mapper in DistCP +distcpMapBandwidth=100 + +##### Email Notification for Falcon instance completion +falcon.recipe.notification.type=email +falcon.recipe.notification.receivers=NA + http://git-wip-us.apache.org/repos/asf/falcon/blob/4a76b11b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hive-disaster-recovery-template.xml ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/resources/HdfsRecipe/hive-disaster-recovery-template.xml b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hive-disaster-recovery-template.xml deleted file mode 100644 index 46546ce..0000000 --- a/falcon-regression/merlin/src/test/resources/HdfsRecipe/hive-disaster-recovery-template.xml +++ /dev/null @@ -1,44 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - ~ Licensed to the Apache Software Foundation (ASF) under one - ~ or more contributor license agreements. See the NOTICE file - ~ distributed with this work for additional information - ~ regarding copyright ownership. The ASF licenses this file - ~ to you under the Apache License, Version 2.0 (the - ~ "License"); you may not use this file except in compliance - ~ with the License. You may obtain a copy of the License at - ~ - ~ http://www.apache.org/licenses/LICENSE-2.0 - ~ - ~ Unless required by applicable law or agreed to in writing, software - ~ distributed under the License is distributed on an "AS IS" BASIS, - ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - ~ See the License for the specific language governing permissions and - ~ limitations under the License. - --> - -<process name="##falcon.recipe.job.name##" xmlns="uri:falcon:process:0.1"> - <clusters> - <!-- source --> - <cluster name="##falcon.recipe.cluster.name##"> - <validity end="##falcon.recipe.cluster.validity.end##" start="##falcon.recipe.cluster.validity.start##"/> - </cluster> - </clusters> - - <tags>_falcon_mirroring_type=HDFS</tags> - - <parallel>1</parallel> - <!-- Dir replication needs to run only once to catch up --> - <order>LAST_ONLY</order> - <frequency>##falcon.recipe.frequency##</frequency> - <timezone>UTC</timezone> - - <properties> - <property name="oozie.wf.subworkflow.classpath.inheritance" value="true"/> - </properties> - - <workflow name="##falcon.recipe.workflow.name##" engine="oozie" path="/apps/data-mirroring/workflows/hdfs-replication-workflow.xml" lib="##workflow.lib.path##"/> - <retry policy="##falcon.recipe.retry.policy##" delay="##falcon.recipe.retry.delay##" attempts="3"/> - <notification type="##falcon.recipe.notification.type##" to="##falcon.recipe.notification.receivers##"/> - <ACL/> -</process> http://git-wip-us.apache.org/repos/asf/falcon/blob/4a76b11b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hive-disaster-recovery-workflow.xml ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/resources/HdfsRecipe/hive-disaster-recovery-workflow.xml b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hive-disaster-recovery-workflow.xml deleted file mode 100644 index aa820d0..0000000 --- a/falcon-regression/merlin/src/test/resources/HdfsRecipe/hive-disaster-recovery-workflow.xml +++ /dev/null @@ -1,82 +0,0 @@ -<!-- - ~ Licensed to the Apache Software Foundation (ASF) under one - ~ or more contributor license agreements. See the NOTICE file - ~ distributed with this work for additional information - ~ regarding copyright ownership. The ASF licenses this file - ~ to you under the Apache License, Version 2.0 (the - ~ "License"); you may not use this file except in compliance - ~ with the License. You may obtain a copy of the License at - ~ - ~ http://www.apache.org/licenses/LICENSE-2.0 - ~ - ~ Unless required by applicable law or agreed to in writing, software - ~ distributed under the License is distributed on an "AS IS" BASIS, - ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - ~ See the License for the specific language governing permissions and - ~ limitations under the License. - --> -<workflow-app xmlns='uri:oozie:workflow:0.3' name='falcon-dr-fs-workflow'> - <start to='dr-replication'/> - <!-- Replication action --> - <action name="dr-replication"> - <java> - <job-tracker>${jobTracker}</job-tracker> - <name-node>${nameNode}</name-node> - <configuration> - <property> <!-- hadoop 2 parameter --> - <name>oozie.launcher.mapreduce.job.user.classpath.first</name> - <value>true</value> - </property> - <property> - <name>mapred.job.queue.name</name> - <value>${queueName}</value> - </property> - <property> - <name>oozie.launcher.mapred.job.priority</name> - <value>${jobPriority}</value> - </property> - <property> - <name>oozie.use.system.libpath</name> - <value>true</value> - </property> - <property> - <name>oozie.action.sharelib.for.java</name> - <value>distcp</value> - </property> - <property> - <name>oozie.launcher.oozie.libpath</name> - <value>${wf:conf("falcon.libpath")}</value> - </property> - <property> - <name>oozie.launcher.mapreduce.job.hdfs-servers</name> - <value>${drSourceClusterFS},${drTargetClusterFS}</value> - </property> - </configuration> - <main-class>org.apache.falcon.replication.FeedReplicator</main-class> - <arg>-Dmapred.job.queue.name=${queueName}</arg> - <arg>-Dmapred.job.priority=${jobPriority}</arg> - <arg>-maxMaps</arg> - <arg>${distcpMaxMaps}</arg> - <arg>-mapBandwidth</arg> - <arg>${distcpMapBandwidth}</arg> - <arg>-sourcePaths</arg> - <arg>${drSourceDir}</arg> - <arg>-targetPath</arg> - <arg>${drTargetClusterFS}${drTargetDir}</arg> - <arg>-falconFeedStorageType</arg> - <arg>FILESYSTEM</arg> - <arg>-availabilityFlag</arg> - <arg>${availabilityFlag == 'NA' ? "NA" : availabilityFlag}</arg> - <arg>-counterLogDir</arg> - <arg>${logDir}/job-${nominalTime}/${srcClusterName == 'NA' ? '' : srcClusterName}</arg> - </java> - <ok to="end"/> - <error to="fail"/> - </action> - <kill name="fail"> - <message> - Workflow action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] - </message> - </kill> - <end name="end"/> -</workflow-app> http://git-wip-us.apache.org/repos/asf/falcon/blob/4a76b11b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hive-disaster-recovery.properties ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/resources/HdfsRecipe/hive-disaster-recovery.properties b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hive-disaster-recovery.properties deleted file mode 100644 index 6c715f3..0000000 --- a/falcon-regression/merlin/src/test/resources/HdfsRecipe/hive-disaster-recovery.properties +++ /dev/null @@ -1,79 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -##### NOTE: This is a TEMPLATE file which can be copied and edited - -##### Recipe properties -##### Unique recipe job name -falcon.recipe.name=hdfs-replication-monthly - -##### Workflow properties -falcon.recipe.workflow.name=hdfs-dr-workflow -# Provide Wf absolute path. This can be HDFS or local FS path. If WF is on local FS it will be copied to HDFS -falcon.recipe.workflow.path=/apps/data-mirroring/workflows/hdfs-replication-workflow.xml - -##### Cluster properties - -# Cluster where job should run -falcon.recipe.cluster.name=primaryCluster -# Change the cluster hdfs write end point here. This is mandatory. -falcon.recipe.cluster.hdfs.writeEndPoint=hdfs://localhost:8020 -# Change the cluster validity start time here -falcon.recipe.cluster.validity.start=2015-03-13T00:00Z -# Change the cluster validity end time here -falcon.recipe.cluster.validity.end=2016-12-30T00:00Z - -##### Scheduling properties -# Change the recipe frequency here. Valid frequency type are minutes, hours, days, months -falcon.recipe.process.frequency=minutes(5) - -##### Retry policy properties - -falcon.recipe.retry.policy=periodic -falcon.recipe.retry.delay=minutes(30) -falcon.recipe.retry.attempts=3 -falcon.recipe.retry.onTimeout=false - -##### Tag properties - An optional list of comma separated tags, Key Value Pairs, separated by comma -##### Uncomment to add tags -#falcon.recipe.tags= - -##### ACL properties - Uncomment and change ACL if authorization is enabled - -#falcon.recipe.acl.owner=ambari-qa -#falcon.recipe.acl.group=users -#falcon.recipe.acl.permission=0x755 -#falcon.recipe.nn.principal=nn/[email protected] - -##### Custom Job properties - -# Specify multiple comma separated source directories -drSourceDir=/user/falcon_qa/dr/test/primaryCluster/input -drSourceClusterFS=hdfs://240.0.0.10:8020 -drTargetDir=/user/falcon_qa/dr/test/backupCluster/input -drTargetClusterFS=hdfs://240.0.0.11:8020 - -# Change it to specify the maximum number of mappers for DistCP -distcpMaxMaps=1 -# Change it to specify the bandwidth in MB for each mapper in DistCP -distcpMapBandwidth=100 - -##### Email Notification for Falcon instance completion -falcon.recipe.notification.type=email -falcon.recipe.notification.receivers=NA - http://git-wip-us.apache.org/repos/asf/falcon/blob/4a76b11b/falcon-regression/merlin/src/test/resources/HiveDrRecipe/hive-disaster-recovery-workflow.xml ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/resources/HiveDrRecipe/hive-disaster-recovery-workflow.xml b/falcon-regression/merlin/src/test/resources/HiveDrRecipe/hive-disaster-recovery-workflow.xml index 72d40a3..66f97c4 100644 --- a/falcon-regression/merlin/src/test/resources/HiveDrRecipe/hive-disaster-recovery-workflow.xml +++ b/falcon-regression/merlin/src/test/resources/HiveDrRecipe/hive-disaster-recovery-workflow.xml @@ -158,6 +158,8 @@ <arg>${drJobName}-${nominalTime}</arg> <arg>-executionStage</arg> <arg>export</arg> + <arg>-counterLogDir</arg> + <arg>${logDir}/job-${nominalTime}/${srcClusterName == 'NA' ? '' : srcClusterName}</arg> </java> <ok to="import-dr-replication"/> <error to="fail"/>
