Author: eli
Date: Thu Aug 2 17:49:52 2012
New Revision: 1368621
URL: http://svn.apache.org/viewvc?rev=1368621&view=rev
Log:
HDFS-3697. Enable fadvise readahead by default. Contributed by Todd Lipcon
Modified:
hadoop/common/branches/branch-1/CHANGES.txt
hadoop/common/branches/branch-1/src/hdfs/hdfs-default.xml
hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DFSConfigKeys.java
Modified: hadoop/common/branches/branch-1/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1368621&r1=1368620&r2=1368621&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1/CHANGES.txt Thu Aug 2 17:49:52 2012
@@ -63,6 +63,8 @@ Release 1.2.0 - unreleased
HDFS-3667. Add retry support to WebHdfsFileSystem. (szetszwo)
+ HDFS-3697. Enable fadvise readahead by default. (todd via eli)
+
OPTIMIZATIONS
HDFS-2533. Backport: Remove needless synchronization on some FSDataSet
Modified: hadoop/common/branches/branch-1/src/hdfs/hdfs-default.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/hdfs-default.xml?rev=1368621&r1=1368620&r2=1368621&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/hdfs/hdfs-default.xml (original)
+++ hadoop/common/branches/branch-1/src/hdfs/hdfs-default.xml Thu Aug 2
17:49:52 2012
@@ -450,6 +450,80 @@ creations/deletions), or "all".</descrip
</property>
<property>
+ <name>dfs.datanode.readahead.bytes</name>
+ <value>4193404</value>
+ <description>
+ While reading block files, if the Hadoop native libraries are
available,
+ the datanode can use the posix_fadvise system call to explicitly
+ page data into the operating system buffer cache ahead of the current
+ reader's position. This can improve performance especially when
+ disks are highly contended.
+
+ This configuration specifies the number of bytes ahead of the current
+ read position which the datanode will attempt to read ahead. This
+ feature may be disabled by configuring this property to 0.
+
+ If the native libraries are not available, this configuration has no
+ effect.
+ </description>
+</property>
+
+<property>
+ <name>dfs.datanode.drop.cache.behind.reads</name>
+ <value>false</value>
+ <description>
+ In some workloads, the data read from HDFS is known to be significantly
+ large enough that it is unlikely to be useful to cache it in the
+ operating system buffer cache. In this case, the DataNode may be
+ configured to automatically purge all data from the buffer cache
+ after it is delivered to the client. This behavior is automatically
+ disabled for workloads which read only short sections of a block
+ (e.g HBase random-IO workloads).
+
+ This may improve performance for some workloads by freeing buffer
+ cache spage usage for more cacheable data.
+
+ If the Hadoop native libraries are not available, this configuration
+ has no effect.
+ </description>
+</property>
+
+<property>
+ <name>dfs.datanode.drop.cache.behind.writes</name>
+ <value>false</value>
+ <description>
+ In some workloads, the data written to HDFS is known to be
significantly
+ large enough that it is unlikely to be useful to cache it in the
+ operating system buffer cache. In this case, the DataNode may be
+ configured to automatically purge all data from the buffer cache
+ after it is written to disk.
+
+ This may improve performance for some workloads by freeing buffer
+ cache spage usage for more cacheable data.
+
+ If the Hadoop native libraries are not available, this configuration
+ has no effect.
+ </description>
+</property>
+
+<property>
+ <name>dfs.datanode.sync.behind.writes</name>
+ <value>false</value>
+ <description>
+ If this configuration is enabled, the datanode will instruct the
+ operating system to enqueue all written data to the disk immediately
+ after it is written. This differs from the usual OS policy which
+ may wait for up to 30 seconds before triggering writeback.
+
+ This may improve performance for some workloads by smoothing the
+ IO profile for data written to disk.
+
+ If the Hadoop native libraries are not available, this configuration
+ has no effect.
+ </description>
+</property>
+
+<property>
<name>dfs.client.use.datanode.hostname</name>
<value>false</value>
<description>Whether clients should use datanode hostnames when
Modified:
hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DFSConfigKeys.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1368621&r1=1368620&r2=1368621&view=diff
==============================================================================
---
hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DFSConfigKeys.java
(original)
+++
hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DFSConfigKeys.java
Thu Aug 2 17:49:52 2012
@@ -53,7 +53,7 @@ public class DFSConfigKeys extends Commo
public static final String DFS_DATANODE_BALANCE_BANDWIDTHPERSEC_KEY =
"dfs.datanode.balance.bandwidthPerSec";
public static final long DFS_DATANODE_BALANCE_BANDWIDTHPERSEC_DEFAULT =
1024*1024;
public static final String DFS_DATANODE_READAHEAD_BYTES_KEY =
"dfs.datanode.readahead.bytes";
- public static final long DFS_DATANODE_READAHEAD_BYTES_DEFAULT = 0;
+ public static final long DFS_DATANODE_READAHEAD_BYTES_DEFAULT = 4 * 1024
* 1024; // 4MB
public static final String DFS_DATANODE_DROP_CACHE_BEHIND_WRITES_KEY =
"dfs.datanode.drop.cache.behind.writes";
public static final boolean DFS_DATANODE_DROP_CACHE_BEHIND_WRITES_DEFAULT =
false;
public static final String DFS_DATANODE_SYNC_BEHIND_WRITES_KEY =
"dfs.datanode.sync.behind.writes";
