Author: jing9
Date: Tue Jan 21 02:00:23 2014
New Revision: 1559901
URL: http://svn.apache.org/r1559901
Log:
HDFS-5744. Serialize information for token managers in protobuf. Contributed by
Haohui Mai.
Modified:
hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-5698.txt
hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java
hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
Modified:
hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-5698.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-5698.txt?rev=1559901&r1=1559900&r2=1559901&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-5698.txt
(original)
+++
hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-5698.txt
Tue Jan 21 02:00:23 2014
@@ -16,3 +16,6 @@ HDFS-5698 subtasks
HDFS-5743. Use protobuf to serialize snapshot information. (jing9)
HDFS-5774. Serialize CachePool directives in protobuf. (Haohui Mai via
jing9)
+
+ HDFS-5744. Serialize information for token managers in protobuf. (Haohui
Mai
+ via jing9)
Modified:
hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java?rev=1559901&r1=1559900&r2=1559901&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java
(original)
+++
hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java
Tue Jan 21 02:00:23 2014
@@ -23,12 +23,16 @@ import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.net.InetSocketAddress;
+import java.util.ArrayList;
import java.util.Iterator;
+import java.util.List;
+import java.util.Map.Entry;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
+import
org.apache.hadoop.hdfs.server.namenode.FsImageProto.SecretManagerSection;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory;
import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase;
@@ -46,6 +50,10 @@ import org.apache.hadoop.security.token.
import
org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager;
import org.apache.hadoop.security.token.delegation.DelegationKey;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.protobuf.ByteString;
+
/**
* A HDFS specific delegation token secret manager.
* The secret manager is responsible for generating and accepting the password
@@ -167,7 +175,45 @@ public class DelegationTokenSecretManage
}
serializerCompat.load(in);
}
-
+
+ public static class SecretManagerState {
+ public final SecretManagerSection section;
+ public final List<SecretManagerSection.DelegationKey> keys;
+ public final List<SecretManagerSection.PersistToken> tokens;
+
+ public SecretManagerState(
+ SecretManagerSection s,
+ List<SecretManagerSection.DelegationKey> keys,
+ List<SecretManagerSection.PersistToken> tokens) {
+ this.section = s;
+ this.keys = keys;
+ this.tokens = tokens;
+ }
+ }
+
+ public synchronized void loadSecretManagerState(SecretManagerState state)
+ throws IOException {
+ Preconditions.checkState(!running,
+ "Can't load state from image in a running SecretManager.");
+
+ currentId = state.section.getCurrentId();
+ delegationTokenSequenceNumber = state.section.getTokenSequenceNumber();
+ for (SecretManagerSection.DelegationKey k : state.keys) {
+ addKey(new DelegationKey(k.getId(), k.getExpiryDate(), k.hasKey() ? k
+ .getKey().toByteArray() : null));
+ }
+
+ for (SecretManagerSection.PersistToken t : state.tokens) {
+ DelegationTokenIdentifier id = new DelegationTokenIdentifier(new Text(
+ t.getOwner()), new Text(t.getRenewer()), new Text(t.getRealUser()));
+ id.setIssueDate(t.getIssueDate());
+ id.setMaxDate(t.getMaxDate());
+ id.setSequenceNumber(t.getSequenceNumber());
+ id.setMasterKeyId(t.getMasterKeyId());
+ addPersistedDelegationToken(id, t.getExpiryDate());
+ }
+ }
+
/**
* Store the current state of the SecretManager for persistence
*
@@ -179,7 +225,43 @@ public class DelegationTokenSecretManage
String sdPath) throws IOException {
serializerCompat.save(out, sdPath);
}
-
+
+ public synchronized SecretManagerState saveSecretManagerState() {
+ SecretManagerSection s = SecretManagerSection.newBuilder()
+ .setCurrentId(currentId)
+ .setTokenSequenceNumber(delegationTokenSequenceNumber)
+ .setNumKeys(allKeys.size()).setNumTokens(currentTokens.size()).build();
+ ArrayList<SecretManagerSection.DelegationKey> keys = Lists
+ .newArrayListWithCapacity(allKeys.size());
+ ArrayList<SecretManagerSection.PersistToken> tokens = Lists
+ .newArrayListWithCapacity(currentTokens.size());
+
+ for (DelegationKey v : allKeys.values()) {
+ SecretManagerSection.DelegationKey.Builder b =
SecretManagerSection.DelegationKey
+ .newBuilder().setId(v.getKeyId()).setExpiryDate(v.getExpiryDate());
+ if (v.getEncodedKey() != null) {
+ b.setKey(ByteString.copyFrom(v.getEncodedKey()));
+ }
+ keys.add(b.build());
+ }
+
+ for (Entry<DelegationTokenIdentifier, DelegationTokenInformation> e :
currentTokens
+ .entrySet()) {
+ DelegationTokenIdentifier id = e.getKey();
+ SecretManagerSection.PersistToken.Builder b =
SecretManagerSection.PersistToken
+ .newBuilder().setOwner(id.getOwner().toString())
+ .setRenewer(id.getRenewer().toString())
+ .setRealUser(id.getRealUser().toString())
+ .setIssueDate(id.getIssueDate()).setMaxDate(id.getMaxDate())
+ .setSequenceNumber(id.getSequenceNumber())
+ .setMasterKeyId(id.getMasterKeyId())
+ .setExpiryDate(e.getValue().getRenewDate());
+ tokens.add(b.build());
+ }
+
+ return new SecretManagerState(s, keys, tokens);
+ }
+
/**
* This method is intended to be used only while reading edit logs.
*
@@ -431,4 +513,5 @@ public class DelegationTokenSecretManage
prog.endStep(Phase.LOADING_FSIMAGE, step);
}
}
+
}
Modified:
hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java?rev=1559901&r1=1559900&r2=1559901&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
(original)
+++
hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
Tue Jan 21 02:00:23 2014
@@ -55,9 +55,11 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.StringTableSection;
import org.apache.hadoop.hdfs.server.namenode.snapshot.FSImageFormatPBSnapshot;
import org.apache.hadoop.hdfs.util.MD5FileUtils;
+import
org.apache.hadoop.hdfs.server.namenode.FsImageProto.SecretManagerSection;
import org.apache.hadoop.io.MD5Hash;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.CompressorStream;
+import org.apache.hadoop.security.UserGroupInformation;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@@ -204,7 +206,7 @@ public final class FSImageFormatProtobuf
String n = s.getName();
switch (SectionName.fromString(n)) {
case NS_INFO:
- loadNameSystemSection(in, s);
+ loadNameSystemSection(in);
break;
case STRING_TABLE:
loadStringTableSection(in);
@@ -227,6 +229,9 @@ public final class FSImageFormatProtobuf
case CACHE_MANAGER:
loadCacheManagerSection(in);
break;
+ case SECRET_MANAGER:
+ loadSecretManagerSection(in);
+ break;
default:
LOG.warn("Unregconized section " + n);
break;
@@ -234,8 +239,7 @@ public final class FSImageFormatProtobuf
}
}
- private void loadNameSystemSection(InputStream in,
- FileSummary.Section sections) throws IOException {
+ private void loadNameSystemSection(InputStream in) throws IOException {
NameSystemSection s = NameSystemSection.parseDelimitedFrom(in);
fsn.setGenerationStampV1(s.getGenstampV1());
fsn.setGenerationStampV2(s.getGenstampV2());
@@ -254,6 +258,23 @@ public final class FSImageFormatProtobuf
}
}
+ private void loadSecretManagerSection(InputStream in) throws IOException {
+ SecretManagerSection s = SecretManagerSection.parseDelimitedFrom(in);
+ int numKeys = s.getNumKeys(), numTokens = s.getNumTokens();
+ ArrayList<SecretManagerSection.DelegationKey> keys = Lists
+ .newArrayListWithCapacity(numKeys);
+ ArrayList<SecretManagerSection.PersistToken> tokens = Lists
+ .newArrayListWithCapacity(numTokens);
+
+ for (int i = 0; i < numKeys; ++i)
+ keys.add(SecretManagerSection.DelegationKey.parseDelimitedFrom(in));
+
+ for (int i = 0; i < numTokens; ++i)
+ tokens.add(SecretManagerSection.PersistToken.parseDelimitedFrom(in));
+
+ fsn.loadSecretManagerState(s, keys, tokens);
+ }
+
private void loadCacheManagerSection(InputStream in) throws IOException {
CacheManagerSection s = CacheManagerSection.parseDelimitedFrom(in);
ArrayList<CachePoolInfoProto> pools = Lists.newArrayListWithCapacity(s
@@ -374,6 +395,7 @@ public final class FSImageFormatProtobuf
saveSnapshots(b);
saveStringTableSection(b);
+ saveSecretManagerSection(b);
saveCacheManagerSection(b);
// Flush the buffered data into the file before appending the header
@@ -385,6 +407,21 @@ public final class FSImageFormatProtobuf
savedDigest = new MD5Hash(digester.digest());
}
+ private void saveSecretManagerSection(FileSummary.Builder summary)
+ throws IOException {
+ final FSNamesystem fsn = context.getSourceNamesystem();
+ DelegationTokenSecretManager.SecretManagerState state = fsn
+ .saveSecretManagerState();
+ state.section.writeDelimitedTo(sectionOutputStream);
+ for (SecretManagerSection.DelegationKey k : state.keys)
+ k.writeDelimitedTo(sectionOutputStream);
+
+ for (SecretManagerSection.PersistToken t : state.tokens)
+ t.writeDelimitedTo(sectionOutputStream);
+
+ commitSection(summary, SectionName.SECRET_MANAGER);
+ }
+
private void saveCacheManagerSection(FileSummary.Builder summary) throws
IOException {
final FSNamesystem fsn = context.getSourceNamesystem();
CacheManager.PersistState state = fsn.getCacheManager().saveState();
@@ -481,6 +518,7 @@ public final class FSImageFormatProtobuf
INODE_DIR("INODE_DIR"),
FILES_UNDERCONSTRUCTION("FILES_UNDERCONSTRUCTION"),
SNAPSHOT_DIFF("SNAPSHOT_DIFF"),
+ SECRET_MANAGER("SECRET_MANAGER"),
CACHE_MANAGER("CACHE_MANAGER");
private static final SectionName[] values = SectionName.values();
Modified:
hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1559901&r1=1559900&r2=1559901&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
(original)
+++
hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
Tue Jan 21 02:00:23 2014
@@ -176,6 +176,7 @@ import org.apache.hadoop.hdfs.security.t
import
org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager.AccessMode;
import
org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import
org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager;
+import
org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager.SecretManagerState;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockCollection;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import
org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
@@ -193,6 +194,8 @@ import org.apache.hadoop.hdfs.server.com
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirType;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.hdfs.server.common.Util;
+import
org.apache.hadoop.hdfs.server.namenode.FsImageProto.SecretManagerSection;
+import
org.apache.hadoop.hdfs.server.namenode.FsImageProto.SecretManagerSection.PersistToken;
import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
import org.apache.hadoop.hdfs.server.namenode.JournalSet.JournalAndStream;
import org.apache.hadoop.hdfs.server.namenode.LeaseManager.Lease;
@@ -6264,6 +6267,10 @@ public class FSNamesystem implements Nam
dtSecretManager.saveSecretManagerStateCompat(out, sdPath);
}
+ SecretManagerState saveSecretManagerState() {
+ return dtSecretManager.saveSecretManagerState();
+ }
+
/**
* @param in load the state of secret manager from input stream
*/
@@ -6271,6 +6278,12 @@ public class FSNamesystem implements Nam
dtSecretManager.loadSecretManagerStateCompat(in);
}
+ void loadSecretManagerState(SecretManagerSection s,
+ List<SecretManagerSection.DelegationKey> keys,
+ List<SecretManagerSection.PersistToken> tokens) throws IOException {
+ dtSecretManager.loadSecretManagerState(new SecretManagerState(s, keys,
tokens));
+ }
+
/**
* Log the updateMasterKey operation to edit logs
*
Modified:
hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto?rev=1559901&r1=1559900&r2=1559901&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
(original)
+++
hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
Tue Jan 21 02:00:23 2014
@@ -239,6 +239,31 @@ message StringTableSection {
// repeated Entry
}
+message SecretManagerSection {
+ message DelegationKey {
+ optional uint32 id = 1;
+ optional uint64 expiryDate = 2;
+ optional bytes key = 3;
+ }
+ message PersistToken {
+ optional uint32 version = 1;
+ optional string owner = 2;
+ optional string renewer = 3;
+ optional string realUser = 4;
+ optional uint64 issueDate = 5;
+ optional uint64 maxDate = 6;
+ optional uint32 sequenceNumber = 7;
+ optional uint32 masterKeyId = 8;
+ optional uint64 expiryDate = 9;
+ }
+ optional uint32 currentId = 1;
+ optional uint32 tokenSequenceNumber = 2;
+ optional uint32 numKeys = 3;
+ optional uint32 numTokens = 4;
+ // repeated DelegationKey keys
+ // repeated PersistToken tokens
+}
+
message CacheManagerSection {
required uint64 nextDirectiveId = 1;
required uint32 numPools = 2;
@@ -246,3 +271,4 @@ message CacheManagerSection {
// repeated CachePoolInfoProto pools
// repeated CacheDirectiveInfoProto directives
}
+
