Repository: incubator-falcon Updated Branches: refs/heads/master 81904565a -> 394055a54
FALCON-815 Rename mapBandwidthKB to mapBandwidth in DistCp. Contributed by Sowmya Ramesh Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/112f9009 Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/112f9009 Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/112f9009 Branch: refs/heads/master Commit: 112f90094df224e47760300bf768d264cbea11ff Parents: d2a2508 Author: Venkatesh Seetharam <venkat...@apache.org> Authored: Fri Nov 7 20:21:13 2014 -0800 Committer: Venkatesh Seetharam <venkat...@apache.org> Committed: Fri Nov 7 21:33:25 2014 -0800 ---------------------------------------------------------------------- CHANGES.txt | 3 +++ .../src/main/resources/hdfs-replication-workflow.xml | 4 ++-- .../src/main/resources/hdfs-replication.properties | 4 ++-- common/src/main/resources/runtime.properties | 2 +- docs/src/site/twiki/EntitySpecification.twiki | 4 ++-- .../falcon/oozie/feed/FeedReplicationCoordinatorBuilder.java | 4 ++-- oozie/src/main/resources/action/feed/replication-action.xml | 4 ++-- .../apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java | 4 ++-- oozie/src/test/resources/feed/fs-replication-feed.xml | 2 +- .../java/org/apache/falcon/replication/FeedReplicator.java | 6 +++--- .../java/org/apache/falcon/replication/FeedReplicatorTest.java | 6 +++--- 11 files changed, 23 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/112f9009/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index aaea457..7abc820 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -129,6 +129,9 @@ Trunk (Unreleased) OPTIMIZATIONS BUG FIXES + FALCON-815 Rename mapBandwidthKB to mapBandwidth in DistCp + (Sowmya Ramesh via Venkatesh Seetharam) + FALCON-595 Improvements to DefaultAuthorizationProvider code (Raghav Kumar Gautam via Venkatesh Seetharam) http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/112f9009/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-workflow.xml ---------------------------------------------------------------------- diff --git a/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-workflow.xml b/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-workflow.xml index 841b9df..4ef8fc7 100644 --- a/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-workflow.xml +++ b/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-workflow.xml @@ -49,8 +49,8 @@ <arg>-Dmapred.job.priority=${jobPriority}</arg> <arg>-maxMaps</arg> <arg>${maxMaps}</arg> - <arg>-mapBandwidthKB</arg> - <arg>${mapBandwidthKB}</arg> + <arg>-mapBandwidth</arg> + <arg>${mapBandwidth}</arg> <arg>-sourcePaths</arg> <arg>${nameNode}${drSourceDir}</arg> <arg>-targetPath</arg> http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/112f9009/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication.properties ---------------------------------------------------------------------- diff --git a/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication.properties b/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication.properties index 29d6cab..19b8459 100644 --- a/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication.properties +++ b/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication.properties @@ -61,5 +61,5 @@ falcon.recipe.process.property5.name=drTargetCluster falcon.recipe.process.property5.value=backupCluster falcon.recipe.process.property6.name=maxMaps falcon.recipe.process.property6.value=5 -falcon.recipe.process.property7.name=mapBandwidthKB -falcon.recipe.process.property7.value=102400 +falcon.recipe.process.property7.name=mapBandwidth +falcon.recipe.process.property7.value=100 http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/112f9009/common/src/main/resources/runtime.properties ---------------------------------------------------------------------- diff --git a/common/src/main/resources/runtime.properties b/common/src/main/resources/runtime.properties index 185602c..1d35667 100644 --- a/common/src/main/resources/runtime.properties +++ b/common/src/main/resources/runtime.properties @@ -27,4 +27,4 @@ *.falcon.parentworkflow.retry.interval.secs=1 *.falcon.replication.workflow.maxmaps=5 -*.falcon.replication.workflow.mapbandwidthKB=102400 +*.falcon.replication.workflow.mapbandwidth=100 http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/112f9009/docs/src/site/twiki/EntitySpecification.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/EntitySpecification.twiki b/docs/src/site/twiki/EntitySpecification.twiki index 1e603ec..a81a626 100644 --- a/docs/src/site/twiki/EntitySpecification.twiki +++ b/docs/src/site/twiki/EntitySpecification.twiki @@ -278,13 +278,13 @@ permission indicates the permission. <property name="timeout" value="hours(1)"/> <property name="parallel" value="3"/> <property name="maxMaps" value="8"/> - <property name="mapBandwidthKB" value="1024"/> + <property name="mapBandwidth" value="1"/> </properties> </verbatim> A key-value pair, which are propagated to the workflow engine. "queueName" and "jobPriority" are special properties available to user to specify the Hadoop job queue and priority, the same value is used by Falcons launcher job. "timeout" and "parallel" are other special properties which decides replication instance's timeout value while -waiting for the feed instance and parallel decides the concurrent replication instances that can run at any given time. "maxMaps" represents the maximum number of maps used during replication. "mapBandwidthKB" represents the bandwidth in KB/s used by each mapper during replication. +waiting for the feed instance and parallel decides the concurrent replication instances that can run at any given time. "maxMaps" represents the maximum number of maps used during replication. "mapBandwidth" represents the bandwidth in MB/s used by each mapper during replication. ---++ Process Specification A process defines configuration for a workflow. A workflow is a directed acyclic graph(DAG) which defines the job for the workflow engine. A process definition defines the configurations required to run the workflow job. For example, process defines the frequency at which the workflow should run, the clusters on which the workflow should run, the inputs and outputs for the workflow, how the workflow failures should be handled, how the late inputs should be handled and so on. http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/112f9009/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedReplicationCoordinatorBuilder.java ---------------------------------------------------------------------- diff --git a/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedReplicationCoordinatorBuilder.java b/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedReplicationCoordinatorBuilder.java index c5366dc..8f7f01a 100644 --- a/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedReplicationCoordinatorBuilder.java +++ b/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedReplicationCoordinatorBuilder.java @@ -73,7 +73,7 @@ public class FeedReplicationCoordinatorBuilder extends OozieCoordinatorBuilder<F private static final String PARALLEL = "parallel"; private static final String TIMEOUT = "timeout"; private static final String MR_MAX_MAPS = "maxMaps"; - private static final String MR_MAP_BANDWIDTH = "mapBandwidthKB"; + private static final String MR_MAP_BANDWIDTH = "mapBandwidth"; public FeedReplicationCoordinatorBuilder(Feed entity) { super(entity, LifeCycle.REPLICATION); @@ -205,7 +205,7 @@ public class FeedReplicationCoordinatorBuilder extends OozieCoordinatorBuilder<F } private String getDefaultMapBandwidth() { - return RuntimeProperties.get().getProperty("falcon.replication.workflow.mapbandwidthKB", "102400"); + return RuntimeProperties.get().getProperty("falcon.replication.workflow.mapbandwidth", "100"); } private String getPathsWithPartitions(Cluster srcCluster, Cluster trgCluster) throws FalconException { http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/112f9009/oozie/src/main/resources/action/feed/replication-action.xml ---------------------------------------------------------------------- diff --git a/oozie/src/main/resources/action/feed/replication-action.xml b/oozie/src/main/resources/action/feed/replication-action.xml index 5d154b4..da40b74 100644 --- a/oozie/src/main/resources/action/feed/replication-action.xml +++ b/oozie/src/main/resources/action/feed/replication-action.xml @@ -44,8 +44,8 @@ <arg>-Dmapred.job.priority=${jobPriority}</arg> <arg>-maxMaps</arg> <arg>${maxMaps}</arg> - <arg>-mapBandwidthKB</arg> - <arg>${mapBandwidthKB}</arg> + <arg>-mapBandwidth</arg> + <arg>${mapBandwidth}</arg> <arg>-sourcePaths</arg> <arg>${distcpSourcePaths}</arg> <arg>-targetPath</arg> http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/112f9009/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java ---------------------------------------------------------------------- diff --git a/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java b/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java index 74cc509..42c231f 100644 --- a/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java +++ b/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java @@ -229,7 +229,7 @@ public class OozieFeedWorkflowBuilderTest extends AbstractTestBase { Assert.assertEquals(props.get("queueName"), "default"); Assert.assertEquals(props.get("jobPriority"), "NORMAL"); Assert.assertEquals(props.get("maxMaps"), "5"); - Assert.assertEquals(props.get("mapBandwidthKB"), "102400"); + Assert.assertEquals(props.get("mapBandwidth"), "100"); assertLibExtensions(coord, "replication"); WORKFLOWAPP wf = getWorkflowapp(trgMiniDFS.getFileSystem(), coord); @@ -336,7 +336,7 @@ public class OozieFeedWorkflowBuilderTest extends AbstractTestBase { Assert.assertEquals(props.get("distcpTargetPaths"), "${coord:dataOut('output')}"); Assert.assertEquals(props.get("falconFeedStorageType"), Storage.TYPE.FILESYSTEM.name()); Assert.assertEquals(props.get("maxMaps"), "33"); - Assert.assertEquals(props.get("mapBandwidthKB"), "2048"); + Assert.assertEquals(props.get("mapBandwidth"), "2"); verifyEntityProperties(aFeed, aCluster, srcCluster, WorkflowExecutionContext.EntityOperations.REPLICATE, props); http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/112f9009/oozie/src/test/resources/feed/fs-replication-feed.xml ---------------------------------------------------------------------- diff --git a/oozie/src/test/resources/feed/fs-replication-feed.xml b/oozie/src/test/resources/feed/fs-replication-feed.xml index bada507..0e9065c 100644 --- a/oozie/src/test/resources/feed/fs-replication-feed.xml +++ b/oozie/src/test/resources/feed/fs-replication-feed.xml @@ -63,6 +63,6 @@ <schema provider="protobuf" location="/databus/streams_local/click_rr/schema/"/> <properties> <property name="maxMaps" value="33" /> - <property name="mapBandwidthKB" value="2048" /> + <property name="mapBandwidth" value="2" /> </properties> </feed> http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/112f9009/replication/src/main/java/org/apache/falcon/replication/FeedReplicator.java ---------------------------------------------------------------------- diff --git a/replication/src/main/java/org/apache/falcon/replication/FeedReplicator.java b/replication/src/main/java/org/apache/falcon/replication/FeedReplicator.java index 2bf2a98..90ac753 100644 --- a/replication/src/main/java/org/apache/falcon/replication/FeedReplicator.java +++ b/replication/src/main/java/org/apache/falcon/replication/FeedReplicator.java @@ -89,8 +89,8 @@ public class FeedReplicator extends Configured implements Tool { opt.setRequired(true); options.addOption(opt); - opt = new Option("mapBandwidthKB", true, - "bandwidth per map (in KB) to use for this copy"); + opt = new Option("mapBandwidth", true, + "bandwidth per map (in MB) to use for this copy"); opt.setRequired(true); options.addOption(opt); @@ -119,7 +119,7 @@ public class FeedReplicator extends Configured implements Tool { distcpOptions.setSyncFolder(true); distcpOptions.setBlocking(true); distcpOptions.setMaxMaps(Integer.valueOf(cmd.getOptionValue("maxMaps"))); - distcpOptions.setMapBandwidth(Integer.valueOf(cmd.getOptionValue("mapBandwidthKB"))); + distcpOptions.setMapBandwidth(Integer.valueOf(cmd.getOptionValue("mapBandwidth"))); return distcpOptions; } http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/112f9009/replication/src/test/java/org/apache/falcon/replication/FeedReplicatorTest.java ---------------------------------------------------------------------- diff --git a/replication/src/test/java/org/apache/falcon/replication/FeedReplicatorTest.java b/replication/src/test/java/org/apache/falcon/replication/FeedReplicatorTest.java index d64eb61..539d00d 100644 --- a/replication/src/test/java/org/apache/falcon/replication/FeedReplicatorTest.java +++ b/replication/src/test/java/org/apache/falcon/replication/FeedReplicatorTest.java @@ -38,14 +38,14 @@ public class FeedReplicatorTest { * <arg>-update</arg> * <arg>-blocking</arg><arg>true</arg> * <arg>-maxMaps</arg><arg>3</arg> - * <arg>-mapBandwidthKB</arg><arg>4</arg> + * <arg>-mapBandwidth</arg><arg>4</arg> * <arg>-sourcePaths</arg><arg>${distcpSourcePaths}</arg> * <arg>-targetPath</arg><arg>${distcpTargetPaths}</arg> */ final String[] args = { "true", "-maxMaps", "3", - "-mapBandwidthKB", "4096", + "-mapBandwidth", "4", "-sourcePaths", "hdfs://localhost:8020/tmp/", "-targetPath", "hdfs://localhost1:8020/tmp/", "-falconFeedStorageType", Storage.TYPE.FILESYSTEM.name(), @@ -58,7 +58,7 @@ public class FeedReplicatorTest { List<Path> srcPaths = new ArrayList<Path>(); srcPaths.add(new Path("hdfs://localhost:8020/tmp/")); Assert.assertEquals(options.getMaxMaps(), 3); - Assert.assertEquals(options.getMapBandwidth(), 4096); + Assert.assertEquals(options.getMapBandwidth(), 4); Assert.assertEquals(options.getSourcePaths(), srcPaths); Assert.assertEquals(options.getTargetPath(), new Path("hdfs://localhost1:8020/tmp/")); }
