Author: jing9
Date: Tue May 13 18:41:07 2014
New Revision: 1594324
URL: http://svn.apache.org/r1594324
Log:
HDFS-6186. Merge change r1594314 from trunk.
Added:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java
- copied unchanged from r1594314,
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1594324&r1=1594323&r2=1594324&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Tue May 13 18:41:07 2014
@@ -100,6 +100,8 @@ Release 2.5.0 - UNRELEASED
HDFS-6230. Expose upgrade status through NameNode web UI.
(Mit Desai via wheat9)
+ HDFS-6186. Pause deletion of blocks when the namenode starts up. (jing9)
+
OPTIMIZATIONS
HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn)
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1594324&r1=1594323&r2=1594324&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
Tue May 13 18:41:07 2014
@@ -245,6 +245,10 @@ public class DFSConfigKeys extends Commo
"dfs.namenode.path.based.cache.refresh.interval.ms";
public static final long
DFS_NAMENODE_PATH_BASED_CACHE_REFRESH_INTERVAL_MS_DEFAULT = 30000L;
+ /** Pending period of block deletion since NameNode startup */
+ public static final String DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_MS_KEY
= "dfs.namenode.startup.delay.block.deletion.ms";
+ public static final long
DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_MS_DEFAULT = 0L;
+
// Whether to enable datanode's stale state detection and usage for reads
public static final String DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_KEY =
"dfs.namenode.avoid.read.stale.datanode";
public static final boolean
DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_DEFAULT = false;
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java?rev=1594324&r1=1594323&r2=1594324&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
Tue May 13 18:41:07 2014
@@ -261,7 +261,11 @@ public class BlockManager {
this.namesystem = namesystem;
datanodeManager = new DatanodeManager(this, namesystem, conf);
heartbeatManager = datanodeManager.getHeartbeatManager();
- invalidateBlocks = new InvalidateBlocks(datanodeManager);
+
+ final long pendingPeriod = conf.getLong(
+ DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_MS_KEY,
+ DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_MS_DEFAULT);
+ invalidateBlocks = new InvalidateBlocks(datanodeManager, pendingPeriod);
// Compute the map capacity by allocating 2% of total memory
blocksMap = new BlocksMap(
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java?rev=1594324&r1=1594323&r2=1594324&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java
Tue May 13 18:41:07 2014
@@ -18,18 +18,24 @@
package org.apache.hadoop.hdfs.server.blockmanagement;
import java.io.PrintWriter;
+import java.text.SimpleDateFormat;
import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.Iterator;
+import java.util.Calendar;
+import java.util.GregorianCalendar;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
+import org.apache.commons.logging.Log;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.util.LightWeightHashSet;
+import org.apache.hadoop.util.Time;
+
+import com.google.common.annotations.VisibleForTesting;
/**
* Keeps a Collection for every named machine containing blocks
@@ -46,8 +52,28 @@ class InvalidateBlocks {
private final DatanodeManager datanodeManager;
- InvalidateBlocks(final DatanodeManager datanodeManager) {
+ /**
+ * The period of pending time for block invalidation since the NameNode
+ * startup
+ */
+ private final long pendingPeriodInMs;
+ /** the startup time */
+ private final long startupTime = Time.monotonicNow();
+
+ InvalidateBlocks(final DatanodeManager datanodeManager, long
pendingPeriodInMs) {
this.datanodeManager = datanodeManager;
+ this.pendingPeriodInMs = pendingPeriodInMs;
+ printBlockDeletionTime(BlockManager.LOG);
+ }
+
+ private void printBlockDeletionTime(final Log log) {
+ log.info(DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_MS_KEY
+ + " is set to " + pendingPeriodInMs + " ms.");
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy MMM dd HH:mm:ss");
+ Calendar calendar = new GregorianCalendar();
+ calendar.add(Calendar.SECOND, (int) (this.pendingPeriodInMs / 1000));
+ log.info("The block deletion will start around "
+ + sdf.format(calendar.getTime()));
}
/** @return the number of blocks to be invalidated . */
@@ -136,8 +162,25 @@ class InvalidateBlocks {
return new ArrayList<String>(node2blocks.keySet());
}
+ /**
+ * @return the remianing pending time
+ */
+ @VisibleForTesting
+ long getInvalidationDelay() {
+ return pendingPeriodInMs - (Time.monotonicNow() - startupTime);
+ }
+
/** Invalidate work for the storage. */
int invalidateWork(final String storageId) {
+ final long delay = getInvalidationDelay();
+ if (delay > 0) {
+ if (BlockManager.LOG.isDebugEnabled()) {
+ BlockManager.LOG
+ .debug("Block deletion is delayed during NameNode startup. "
+ + "The deletion will start after " + delay + " ms.");
+ }
+ return 0;
+ }
final DatanodeDescriptor dn = datanodeManager.getDatanode(storageId);
if (dn == null) {
remove(storageId);
