Author: todd
Date: Thu Aug 4 17:09:14 2011
New Revision: 1153931
URL: http://svn.apache.org/viewvc?rev=1153931&view=rev
Log:
Merge trunk into HA branch after mavenization of hadoop-common
(no conflicts, straight SVN merge)
Modified:
hadoop/common/branches/HDFS-1623/hdfs/ (props changed)
hadoop/common/branches/HDFS-1623/hdfs/CHANGES.txt
hadoop/common/branches/HDFS-1623/hdfs/ivy.xml
hadoop/common/branches/HDFS-1623/hdfs/src/c++/libhdfs/ (props changed)
hadoop/common/branches/HDFS-1623/hdfs/src/java/ (props changed)
hadoop/common/branches/HDFS-1623/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
hadoop/common/branches/HDFS-1623/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
hadoop/common/branches/HDFS-1623/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/HDFS-1623/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/ (props changed)
hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBlockTokenWithDFS.java
hadoop/common/branches/HDFS-1623/hdfs/src/webapps/datanode/ (props
changed)
hadoop/common/branches/HDFS-1623/hdfs/src/webapps/hdfs/ (props changed)
hadoop/common/branches/HDFS-1623/hdfs/src/webapps/secondary/ (props
changed)
Propchange: hadoop/common/branches/HDFS-1623/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug 4 17:09:14 2011
@@ -1,3 +1,4 @@
+/hadoop/common/trunk/hdfs:1152502-1153927
/hadoop/core/branches/branch-0.19/hdfs:713112
/hadoop/hdfs/branches/HDFS-1052:987665-1095512
/hadoop/hdfs/branches/HDFS-265:796829-820463
Modified: hadoop/common/branches/HDFS-1623/hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hdfs/CHANGES.txt?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-1623/hdfs/CHANGES.txt Thu Aug 4 17:09:14 2011
@@ -626,6 +626,9 @@ Trunk (unreleased changes)
HDFS-2212. Refactor double-buffering code out of EditLogOutputStreams.
(todd via eli)
+ HDFS-2199. Move blockTokenSecretManager from FSNamesystem to BlockManager.
+ (Uma Maheswara Rao G via szetszwo)
+
OPTIMIZATIONS
HDFS-1458. Improve checkpoint performance by avoiding unnecessary image
@@ -911,6 +914,9 @@ Trunk (unreleased changes)
HDFS-1776. Bug in Concat code. (Bharath Mundlapudi via Dmytro Molkov)
+ HDFS-2196. Make ant build system work with hadoop-common JAR generated
+ by Maven. (Alejandro Abdelnur via tomwhite)
+
BREAKDOWN OF HDFS-1073 SUBTASKS
HDFS-1521. Persist transaction ID on disk between NN restarts.
Modified: hadoop/common/branches/HDFS-1623/hdfs/ivy.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hdfs/ivy.xml?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hdfs/ivy.xml (original)
+++ hadoop/common/branches/HDFS-1623/hdfs/ivy.xml Thu Aug 4 17:09:14 2011
@@ -58,6 +58,7 @@
</publications>
<dependencies>
+ <dependency org="org.apache.hadoop" name="hadoop-annotations"
rev="${hadoop-common.version}" conf="compile->default"/>
<dependency org="org.apache.hadoop" name="hadoop-common"
rev="${hadoop-common.version}" conf="common->default">
<exclude module="ant"/>
</dependency>
@@ -66,7 +67,11 @@
</dependency>
<dependency org="commons-logging" name="commons-logging"
rev="${commons-logging.version}" conf="compile->master"/>
<dependency org="commons-daemon" name="commons-daemon"
rev="${commons-daemon.version}" conf="hdfs->default" />
- <dependency org="log4j" name="log4j" rev="${log4j.version}"
conf="common->master"/>
+ <dependency org="log4j" name="log4j" rev="${log4j.version}"
conf="common->master">
+ <exclude org="com.sun.jdmk"/>
+ <exclude org="com.sun.jmx"/>
+ <exclude org="javax.jms"/>
+ </dependency>
<dependency org="com.google.guava" name="guava" rev="${guava.version}"
conf="hdfs->default" />
<dependency org="com.google.protobuf" name="protobuf-java" rev="2.4.0a"
conf="common->master"/>
<dependency org="org.apache.hadoop" name="avro" rev="${avro.version}"
conf="compile->master">
Propchange: hadoop/common/branches/HDFS-1623/hdfs/src/c++/libhdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug 4 17:09:14 2011
@@ -1,3 +1,4 @@
+/hadoop/common/trunk/hdfs/src/c++/libhdfs:1152502-1153927
/hadoop/core/branches/branch-0.19/mapred/src/c++/libhdfs:713112
/hadoop/core/trunk/src/c++/libhdfs:776175-784663
/hadoop/hdfs/branches/HDFS-1052/src/c++/libhdfs:987665-1095512
Propchange: hadoop/common/branches/HDFS-1623/hdfs/src/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug 4 17:09:14 2011
@@ -1,3 +1,4 @@
+/hadoop/common/trunk/hdfs/src/java:1152502-1153927
/hadoop/core/branches/branch-0.19/hdfs/src/java:713112
/hadoop/core/trunk/src/hdfs:776175-785643,785929-786278
/hadoop/hdfs/branches/HDFS-1052/src/java:987665-1095512
Modified:
hadoop/common/branches/HDFS-1623/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-1623/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
(original)
+++
hadoop/common/branches/HDFS-1623/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
Thu Aug 4 17:09:14 2011
@@ -24,6 +24,7 @@ import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -47,6 +48,9 @@ import org.apache.hadoop.hdfs.protocol.D
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.UnregisteredNodeException;
+import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
+import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager;
+import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
import
org.apache.hadoop.hdfs.server.blockmanagement.UnderReplicatedBlocks.BlockIterator;
import org.apache.hadoop.hdfs.server.common.HdfsConstants.BlockUCState;
import org.apache.hadoop.hdfs.server.common.HdfsConstants.ReplicaState;
@@ -58,6 +62,7 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations;
import
org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations.BlockWithLocations;
import org.apache.hadoop.net.Node;
+import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.Daemon;
/**
@@ -80,7 +85,26 @@ public class BlockManager {
public volatile long scheduledReplicationBlocksCount = 0L;
private volatile long excessBlocksCount = 0L;
private volatile long pendingDeletionBlocksCount = 0L;
+ private boolean isBlockTokenEnabled;
+ private long blockKeyUpdateInterval;
+ private long blockTokenLifetime;
+ private BlockTokenSecretManager blockTokenSecretManager;
+
+ /** returns the isBlockTokenEnabled - true if block token enabled ,else
false */
+ public boolean isBlockTokenEnabled() {
+ return isBlockTokenEnabled;
+ }
+
+ /** get the block key update interval */
+ public long getBlockKeyUpdateInterval() {
+ return blockKeyUpdateInterval;
+ }
+ /** get the BlockTokenSecretManager */
+ public BlockTokenSecretManager getBlockTokenSecretManager() {
+ return blockTokenSecretManager;
+ }
+
/** Used by metrics */
public long getPendingReplicationBlocksCount() {
return pendingReplicationBlocksCount;
@@ -169,7 +193,43 @@ public class BlockManager {
/** for block replicas placement */
private BlockPlacementPolicy blockplacement;
+
+ /**
+ * Get access keys
+ *
+ * @return current access keys
+ */
+ public ExportedBlockKeys getBlockKeys() {
+ return isBlockTokenEnabled ? blockTokenSecretManager.exportKeys()
+ : ExportedBlockKeys.DUMMY_KEYS;
+ }
+
+ /** Generate block token for a LocatedBlock. */
+ public void setBlockToken(LocatedBlock l) throws IOException {
+ Token<BlockTokenIdentifier> token = blockTokenSecretManager.generateToken(l
+ .getBlock(), EnumSet.of(BlockTokenSecretManager.AccessMode.READ));
+ l.setBlockToken(token);
+ }
+ /** Generate block tokens for the blocks to be returned. */
+ public void setBlockTokens(List<LocatedBlock> locatedBlocks) throws
IOException {
+ for(LocatedBlock l : locatedBlocks) {
+ setBlockToken(l);
+ }
+ }
+
+ /**
+ * Update access keys.
+ */
+ public void updateBlockKey() throws IOException {
+ this.blockTokenSecretManager.updateKeys();
+ synchronized (namesystem.heartbeats) {
+ for (DatanodeDescriptor nodeInfo : namesystem.heartbeats) {
+ nodeInfo.needKeyUpdate = true;
+ }
+ }
+ }
+
public BlockManager(FSNamesystem fsn, Configuration conf) throws IOException
{
namesystem = fsn;
datanodeManager = new DatanodeManager(fsn, conf);
@@ -179,7 +239,26 @@ public class BlockManager {
pendingReplications = new PendingReplicationBlocks(conf.getInt(
DFSConfigKeys.DFS_NAMENODE_REPLICATION_PENDING_TIMEOUT_SEC_KEY,
DFSConfigKeys.DFS_NAMENODE_REPLICATION_PENDING_TIMEOUT_SEC_DEFAULT) *
1000L);
-
+ this.isBlockTokenEnabled = conf.getBoolean(
+ DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY,
+ DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_DEFAULT);
+ if (isBlockTokenEnabled) {
+ if (isBlockTokenEnabled) {
+ this.blockKeyUpdateInterval = conf.getLong(
+ DFSConfigKeys.DFS_BLOCK_ACCESS_KEY_UPDATE_INTERVAL_KEY,
+ DFSConfigKeys.DFS_BLOCK_ACCESS_KEY_UPDATE_INTERVAL_DEFAULT) * 60 *
1000L; // 10 hrs
+ this.blockTokenLifetime = conf.getLong(
+ DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_LIFETIME_KEY,
+ DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_LIFETIME_DEFAULT) * 60 *
1000L; // 10 hrs
+ }
+
+ blockTokenSecretManager = new BlockTokenSecretManager(true,
+ blockKeyUpdateInterval, blockTokenLifetime);
+ }
+ LOG.info("isBlockTokenEnabled=" + isBlockTokenEnabled
+ + " blockKeyUpdateInterval=" + blockKeyUpdateInterval / (60 * 1000)
+ + " min(s), blockTokenLifetime=" + blockTokenLifetime / (60 * 1000)
+ + " min(s)");
this.maxCorruptFilesReturned = conf.getInt(
DFSConfigKeys.DFS_DEFAULT_MAX_CORRUPT_FILES_RETURNED_KEY,
DFSConfigKeys.DFS_DEFAULT_MAX_CORRUPT_FILES_RETURNED);
Modified:
hadoop/common/branches/HDFS-1623/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-1623/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
(original)
+++
hadoop/common/branches/HDFS-1623/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
Thu Aug 4 17:09:14 2011
@@ -469,7 +469,7 @@ public class DatanodeManager {
nodeReg.getInfoPort(),
nodeReg.getIpcPort());
nodeReg.updateRegInfo(dnReg);
- nodeReg.exportedKeys = namesystem.getBlockKeys();
+ nodeReg.exportedKeys = namesystem.getBlockManager().getBlockKeys();
NameNode.stateChangeLog.info("BLOCK* NameSystem.registerDatanode: "
+ "node registration from " + nodeReg.getName()
Modified:
hadoop/common/branches/HDFS-1623/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-1623/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
(original)
+++
hadoop/common/branches/HDFS-1623/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
Thu Aug 4 17:09:14 2011
@@ -43,9 +43,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import java.util.NavigableMap;
import java.util.Set;
-import java.util.TreeMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -90,9 +88,7 @@ import org.apache.hadoop.hdfs.protocol.Q
import org.apache.hadoop.hdfs.protocol.RecoveryInProgressException;
import org.apache.hadoop.hdfs.protocol.UnregisteredNodeException;
import org.apache.hadoop.hdfs.protocol.datatransfer.ReplaceDatanodeOnFailure;
-import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager;
-import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
import
org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import
org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
@@ -100,7 +96,6 @@ import org.apache.hadoop.hdfs.server.blo
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
-import
org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.BlockTargetPair;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
import org.apache.hadoop.hdfs.server.blockmanagement.UnderReplicatedBlocks;
import org.apache.hadoop.hdfs.server.common.GenerationStamp;
@@ -112,15 +107,9 @@ import org.apache.hadoop.hdfs.server.com
import org.apache.hadoop.hdfs.server.common.Util;
import org.apache.hadoop.hdfs.server.namenode.LeaseManager.Lease;
import org.apache.hadoop.hdfs.server.namenode.metrics.FSNamesystemMBean;
-import org.apache.hadoop.hdfs.server.protocol.BlockCommand;
-import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand;
-import
org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock;
import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations;
-import
org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations.BlockWithLocations;
import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand;
-import org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
-import org.apache.hadoop.hdfs.server.protocol.DisallowedDatanodeException;
import org.apache.hadoop.hdfs.server.protocol.KeyUpdateCommand;
import org.apache.hadoop.hdfs.server.protocol.NamenodeCommand;
import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration;
@@ -221,10 +210,6 @@ public class FSNamesystem implements FSC
private long capacityTotal = 0L, capacityUsed = 0L, capacityRemaining = 0L;
private long blockPoolUsed = 0L;
private int totalLoad = 0;
- boolean isBlockTokenEnabled;
- BlockTokenSecretManager blockTokenSecretManager;
- private long blockKeyUpdateInterval;
- private long blockTokenLifetime;
// Scan interval is not configurable.
private static final long DELEGATION_TOKEN_REMOVER_SCAN_INTERVAL =
@@ -334,10 +319,6 @@ public class FSNamesystem implements FSC
this.dir = new FSDirectory(fsImage, this, conf);
}
this.safeMode = new SafeModeInfo(conf);
- if (isBlockTokenEnabled) {
- blockTokenSecretManager = new BlockTokenSecretManager(true,
- blockKeyUpdateInterval, blockTokenLifetime);
- }
}
void activateSecretManager() throws IOException {
@@ -499,21 +480,6 @@ public class FSNamesystem implements FSC
this.accessTimePrecision =
conf.getLong(DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY, 0);
this.supportAppends = conf.getBoolean(DFSConfigKeys.DFS_SUPPORT_APPEND_KEY,
DFSConfigKeys.DFS_SUPPORT_APPEND_DEFAULT);
- this.isBlockTokenEnabled = conf.getBoolean(
- DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY,
- DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_DEFAULT);
- if (isBlockTokenEnabled) {
- this.blockKeyUpdateInterval = conf.getLong(
- DFSConfigKeys.DFS_BLOCK_ACCESS_KEY_UPDATE_INTERVAL_KEY,
- DFSConfigKeys.DFS_BLOCK_ACCESS_KEY_UPDATE_INTERVAL_DEFAULT) * 60 *
1000L; // 10 hrs
- this.blockTokenLifetime = conf.getLong(
- DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_LIFETIME_KEY,
- DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_LIFETIME_DEFAULT) * 60 * 1000L;
// 10 hrs
- }
- LOG.info("isBlockTokenEnabled=" + isBlockTokenEnabled
- + " blockKeyUpdateInterval=" + blockKeyUpdateInterval / (60 * 1000)
- + " min(s), blockTokenLifetime=" + blockTokenLifetime / (60 * 1000)
- + " min(s)");
this.dtpReplaceDatanodeOnFailure = ReplaceDatanodeOnFailure.get(conf);
}
@@ -646,15 +612,6 @@ public class FSNamesystem implements FSC
}
}
- /**
- * Get access keys
- *
- * @return current access keys
- */
- public ExportedBlockKeys getBlockKeys() {
- return isBlockTokenEnabled ? blockTokenSecretManager.exportKeys()
- : ExportedBlockKeys.DUMMY_KEYS;
- }
/////////////////////////////////////////////////////////
//
@@ -853,9 +810,9 @@ public class FSNamesystem implements FSC
.getBlockLocation(last, n - last.getNumBytes()) : blockManager
.getBlockLocation(last, n);
- if (isBlockTokenEnabled && needBlockToken) {
- setBlockTokens(locatedblocks);
- setBlockToken(lastBlock);
+ if (blockManager.isBlockTokenEnabled() && needBlockToken) {
+ blockManager.setBlockTokens(locatedblocks);
+ blockManager.setBlockToken(lastBlock);
}
return new LocatedBlocks(n, inode.isUnderConstruction(), locatedblocks,
lastBlock, last.isComplete());
@@ -868,19 +825,6 @@ public class FSNamesystem implements FSC
return new LocatedBlock(getExtendedBlock(b), locations, offset, corrupt);
}
- /** Generate block tokens for the blocks to be returned. */
- private void setBlockTokens(List<LocatedBlock> locatedBlocks) throws
IOException {
- for(LocatedBlock l : locatedBlocks) {
- setBlockToken(l);
- }
- }
-
- /** Generate block token for a LocatedBlock. */
- private void setBlockToken(LocatedBlock l) throws IOException {
- Token<BlockTokenIdentifier> token = blockTokenSecretManager.generateToken(l
- .getBlock(), EnumSet.of(BlockTokenSecretManager.AccessMode.READ));
- l.setBlockToken(token);
- }
/**
* Moves all the blocks from srcs and appends them to trg
@@ -1369,8 +1313,8 @@ public class FSNamesystem implements FSC
LocatedBlock lb =
blockManager.convertLastBlockToUnderConstruction(cons);
- if (lb != null && isBlockTokenEnabled) {
-
lb.setBlockToken(blockTokenSecretManager.generateToken(lb.getBlock(),
+ if (lb != null && blockManager.isBlockTokenEnabled()) {
+
lb.setBlockToken(blockManager.getBlockTokenSecretManager().generateToken(lb.getBlock(),
EnumSet.of(BlockTokenSecretManager.AccessMode.WRITE)));
}
return lb;
@@ -1659,8 +1603,8 @@ public class FSNamesystem implements FSC
// Create next block
LocatedBlock b = new LocatedBlock(getExtendedBlock(newBlock), targets,
fileLength);
- if (isBlockTokenEnabled) {
- b.setBlockToken(blockTokenSecretManager.generateToken(b.getBlock(),
+ if (blockManager.isBlockTokenEnabled()) {
+
b.setBlockToken(blockManager.getBlockTokenSecretManager().generateToken(b.getBlock(),
EnumSet.of(BlockTokenSecretManager.AccessMode.WRITE)));
}
return b;
@@ -1708,8 +1652,8 @@ public class FSNamesystem implements FSC
).chooseTarget(src, numAdditionalNodes, clientnode, chosen, true,
excludes, preferredblocksize);
final LocatedBlock lb = new LocatedBlock(blk, targets);
- if (isBlockTokenEnabled) {
- lb.setBlockToken(blockTokenSecretManager.generateToken(lb.getBlock(),
+ if (blockManager.isBlockTokenEnabled()) {
+
lb.setBlockToken(blockManager.getBlockTokenSecretManager().generateToken(lb.getBlock(),
EnumSet.of(BlockTokenSecretManager.AccessMode.COPY)));
}
return lb;
@@ -2705,8 +2649,8 @@ public class FSNamesystem implements FSC
public void addKeyUpdateCommand(final List<DatanodeCommand> cmds,
final DatanodeDescriptor nodeinfo) {
// check access key update
- if (isBlockTokenEnabled && nodeinfo.needKeyUpdate) {
- cmds.add(new KeyUpdateCommand(blockTokenSecretManager.exportKeys()));
+ if (blockManager.isBlockTokenEnabled() && nodeinfo.needKeyUpdate) {
+ cmds.add(new
KeyUpdateCommand(blockManager.getBlockTokenSecretManager().exportKeys()));
nodeinfo.needKeyUpdate = false;
}
}
@@ -2792,17 +2736,6 @@ public class FSNamesystem implements FSC
}
}
- /**
- * Update access keys.
- */
- void updateBlockKey() throws IOException {
- this.blockTokenSecretManager.updateKeys();
- synchronized (heartbeats) {
- for (DatanodeDescriptor nodeInfo : heartbeats) {
- nodeInfo.needKeyUpdate = true;
- }
- }
- }
/**
* Periodically calls heartbeatCheck() and updateBlockKey()
@@ -2820,8 +2753,9 @@ public class FSNamesystem implements FSC
heartbeatCheck();
lastHeartbeatCheck = now;
}
- if (isBlockTokenEnabled && (lastBlockKeyUpdate +
blockKeyUpdateInterval < now)) {
- updateBlockKey();
+ if (blockManager.isBlockTokenEnabled()
+ && (lastBlockKeyUpdate +
blockManager.getBlockKeyUpdateInterval() < now)) {
+ blockManager.updateBlockKey();
lastBlockKeyUpdate = now;
}
} catch (Exception e) {
@@ -4355,8 +4289,8 @@ public class FSNamesystem implements FSC
// get a new generation stamp and an access token
block.setGenerationStamp(nextGenerationStamp());
locatedBlock = new LocatedBlock(block, new DatanodeInfo[0]);
- if (isBlockTokenEnabled) {
- locatedBlock.setBlockToken(blockTokenSecretManager.generateToken(
+ if (blockManager.isBlockTokenEnabled()) {
+
locatedBlock.setBlockToken(blockManager.getBlockTokenSecretManager().generateToken(
block, EnumSet.of(BlockTokenSecretManager.AccessMode.WRITE)));
}
} finally {
Modified:
hadoop/common/branches/HDFS-1623/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-1623/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
(original)
+++
hadoop/common/branches/HDFS-1623/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
Thu Aug 4 17:09:14 2011
@@ -631,7 +631,7 @@ public class NameNode implements Namenod
@Override // NamenodeProtocol
public ExportedBlockKeys getBlockKeys() throws IOException {
- return namesystem.getBlockKeys();
+ return namesystem.getBlockManager().getBlockKeys();
}
@Override // NamenodeProtocol
Propchange: hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug 4 17:09:14 2011
@@ -1,3 +1,4 @@
+/hadoop/common/trunk/hdfs/src/test/hdfs:1152502-1153927
/hadoop/core/branches/branch-0.19/hdfs/src/test/hdfs:713112
/hadoop/core/trunk/src/test/hdfs:776175-785643
/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs:987665-1095512
Modified:
hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBlockTokenWithDFS.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBlockTokenWithDFS.java?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBlockTokenWithDFS.java
(original)
+++
hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBlockTokenWithDFS.java
Thu Aug 4 17:09:14 2011
@@ -190,7 +190,7 @@ public class TestBlockTokenWithDFS exten
assertEquals(numDataNodes, cluster.getDataNodes().size());
// set a short token lifetime (1 second)
SecurityTestUtil.setBlockTokenLifetime(
- cluster.getNameNode().getNamesystem().blockTokenSecretManager,
1000L);
+
cluster.getNameNode().getNamesystem().getBlockManager().getBlockTokenSecretManager(),
1000L);
Path fileToAppend = new Path(FILE_TO_APPEND);
FileSystem fs = cluster.getFileSystem();
@@ -246,7 +246,7 @@ public class TestBlockTokenWithDFS exten
assertEquals(numDataNodes, cluster.getDataNodes().size());
// set a short token lifetime (1 second)
SecurityTestUtil.setBlockTokenLifetime(
- cluster.getNameNode().getNamesystem().blockTokenSecretManager,
1000L);
+
cluster.getNameNode().getNamesystem().getBlockManager().getBlockTokenSecretManager(),
1000L);
Path fileToWrite = new Path(FILE_TO_WRITE);
FileSystem fs = cluster.getFileSystem();
@@ -294,7 +294,9 @@ public class TestBlockTokenWithDFS exten
assertEquals(numDataNodes, cluster.getDataNodes().size());
// set a short token lifetime (1 second) initially
SecurityTestUtil.setBlockTokenLifetime(
- cluster.getNameNode().getNamesystem().blockTokenSecretManager,
1000L);
+ cluster.getNameNode()
+ .getNamesystem().getBlockManager().getBlockTokenSecretManager(),
+ 1000L);
Path fileToRead = new Path(FILE_TO_READ);
FileSystem fs = cluster.getFileSystem();
createFile(fs, fileToRead);
@@ -349,7 +351,8 @@ public class TestBlockTokenWithDFS exten
tryRead(conf, lblock, false);
// use a valid new token
lblock.setBlockToken(cluster.getNameNode().getNamesystem()
- .blockTokenSecretManager.generateToken(lblock.getBlock(),
+ .getBlockManager().getBlockTokenSecretManager().generateToken(
+ lblock.getBlock(),
EnumSet.of(BlockTokenSecretManager.AccessMode.READ)));
// read should succeed
tryRead(conf, lblock, true);
@@ -357,13 +360,15 @@ public class TestBlockTokenWithDFS exten
ExtendedBlock wrongBlock = new ExtendedBlock(lblock.getBlock()
.getBlockPoolId(), lblock.getBlock().getBlockId() + 1);
lblock.setBlockToken(cluster.getNameNode().getNamesystem()
- .blockTokenSecretManager.generateToken(wrongBlock,
+
.getBlockManager().getBlockTokenSecretManager().generateToken(wrongBlock,
EnumSet.of(BlockTokenSecretManager.AccessMode.READ)));
// read should fail
tryRead(conf, lblock, false);
// use a token with wrong access modes
lblock.setBlockToken(cluster.getNameNode().getNamesystem()
- .blockTokenSecretManager.generateToken(lblock.getBlock(), EnumSet.of(
+ .getBlockManager().getBlockTokenSecretManager().generateToken(
+ lblock.getBlock(),
+ EnumSet.of(
BlockTokenSecretManager.AccessMode.WRITE,
BlockTokenSecretManager.AccessMode.COPY,
BlockTokenSecretManager.AccessMode.REPLACE)));
@@ -372,7 +377,9 @@ public class TestBlockTokenWithDFS exten
// set a long token lifetime for future tokens
SecurityTestUtil.setBlockTokenLifetime(
- cluster.getNameNode().getNamesystem().blockTokenSecretManager, 600 *
1000L);
+ cluster.getNameNode()
+ .getNamesystem().getBlockManager().getBlockTokenSecretManager(),
+ 600 * 1000L);
/*
* testing that when cached tokens are expired, DFSClient will re-fetch
Propchange: hadoop/common/branches/HDFS-1623/hdfs/src/webapps/datanode/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug 4 17:09:14 2011
@@ -1,3 +1,4 @@
+/hadoop/common/trunk/hdfs/src/webapps/datanode:1152502-1153927
/hadoop/core/branches/branch-0.19/hdfs/src/webapps/datanode:713112
/hadoop/core/trunk/src/webapps/datanode:776175-784663
/hadoop/hdfs/branches/HDFS-1052/src/webapps/datanode:987665-1095512
Propchange: hadoop/common/branches/HDFS-1623/hdfs/src/webapps/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug 4 17:09:14 2011
@@ -1,3 +1,4 @@
+/hadoop/common/trunk/hdfs/src/webapps/hdfs:1152502-1153927
/hadoop/core/branches/branch-0.19/hdfs/src/webapps/hdfs:713112
/hadoop/core/trunk/src/webapps/hdfs:776175-784663
/hadoop/hdfs/branches/HDFS-1052/src/webapps/hdfs:987665-1095512
Propchange: hadoop/common/branches/HDFS-1623/hdfs/src/webapps/secondary/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug 4 17:09:14 2011
@@ -1,3 +1,4 @@
+/hadoop/common/trunk/hdfs/src/webapps/secondary:1152502-1153927
/hadoop/core/branches/branch-0.19/hdfs/src/webapps/secondary:713112
/hadoop/core/trunk/src/webapps/secondary:776175-784663
/hadoop/hdfs/branches/HDFS-1052/src/webapps/secondary:987665-1095512
