Author: dhruba
Date: Thu Sep 30 13:22:28 2010
New Revision: 1003058
URL: http://svn.apache.org/viewvc?rev=1003058&view=rev
Log:
MAPREDUCE-1816. HAR files used for RAID parity-bite have configurable
partfile size. (Ramkumar Vadali via dhruba)
Modified:
hadoop/mapreduce/trunk/CHANGES.txt
hadoop/mapreduce/trunk/src/contrib/raid/README
hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/ConfigManager.java
hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java
Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=1003058&r1=1003057&r2=1003058&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Thu Sep 30 13:22:28 2010
@@ -316,6 +316,9 @@ Trunk (unreleased changes)
MAPREDUCE-2029. DistributedRaidFileSystem removes itself from FileSystem
cache when it is closed. (Ramkumar Vadali via dhruba)
+ MAPREDUCE-1816. HAR files used for RAID parity-bite have configurable
+ partfile size. (Ramkumar Vadali via dhruba)
+
Release 0.21.0 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/mapreduce/trunk/src/contrib/raid/README
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/raid/README?rev=1003058&r1=1003057&r2=1003058&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/raid/README (original)
+++ hadoop/mapreduce/trunk/src/contrib/raid/README Thu Sep 30 13:22:28 2010
@@ -94,6 +94,16 @@ The following properties can be set in h
</description>
</property>
+ Specify the size of HAR part-files
+ <property>
+ <name>raid.har.partfile.size</name>
+ <value>4294967296</value>
+ <description>The size of HAR part files that store raid parity
+ files. The default is 4GB. The higher the number the fewer the
+ number of files used to store the HAR archive.
+ </description>
+ </property>
+
Specify RaidNode to not use a map-reduce cluster for raiding files in
parallel.
<property>
<name>fs.raidnode.local</name>
Modified:
hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/ConfigManager.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/ConfigManager.java?rev=1003058&r1=1003057&r2=1003058&view=diff
==============================================================================
---
hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/ConfigManager.java
(original)
+++
hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/ConfigManager.java
Thu Sep 30 13:22:28 2010
@@ -53,6 +53,8 @@ class ConfigManager {
/** Time to wait between successive runs of all policies */
public static final long RESCAN_INTERVAL = 3600 * 1000;
+
+ public static final long HAR_PARTFILE_SIZE = 10 * 1024 * 1024 * 1024l;
/**
* Time to wait after the config file has been modified before reloading it
@@ -68,6 +70,7 @@ class ConfigManager {
private boolean lastReloadAttemptFailed = false;
private long reloadInterval = RELOAD_INTERVAL;
private long periodicity; // time between runs of all policies
+ private long harPartfileSize;
// Reload the configuration
private boolean doReload;
@@ -84,6 +87,7 @@ class ConfigManager {
this.doReload = conf.getBoolean("raid.config.reload", true);
this.reloadInterval = conf.getLong("raid.config.reload.interval",
RELOAD_INTERVAL);
this.periodicity = conf.getLong("raid.policy.rescan.interval",
RESCAN_INTERVAL);
+ this.harPartfileSize = conf.getLong("raid.har.partfile.size",
HAR_PARTFILE_SIZE);
if (configFileName == null) {
String msg = "No raid.config.file given in conf - " +
"the Hadoop Raid utility cannot run. Aborting....";
@@ -297,6 +301,10 @@ class ConfigManager {
public synchronized long getPeriodicity() {
return periodicity;
}
+
+ public synchronized long getHarPartfileSize() {
+ return harPartfileSize;
+ }
/**
* Get a collection of all policies
Modified:
hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java?rev=1003058&r1=1003057&r2=1003058&view=diff
==============================================================================
---
hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java
(original)
+++
hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java
Thu Sep 30 13:22:28 2010
@@ -1446,6 +1446,8 @@ public class RaidNode implements RaidPro
String harFileDst = qualifiedPath.getName() + HAR_SUFFIX;
String harFileSrc = qualifiedPath.getName() + "-" +
rand.nextLong() + "-" + HAR_SUFFIX;
+ // HadoopArchives.HAR_PARTFILE_LABEL is private, so hard-coding the label.
+ conf.setLong("har.partfile.size", configMgr.getHarPartfileSize());
HadoopArchives har = new HadoopArchives(conf);
String[] args = new String[6];
args[0] = "-archiveName";
