Author: brandonli
Date: Wed Jan 15 23:11:01 2014
New Revision: 1558598
URL: http://svn.apache.org/r1558598
Log:
HDFS-5768. Consolidate the serialization code in DelegationTokenSecretManager.
Contributed by Haohui Mai
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1558598&r1=1558597&r2=1558598&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Wed Jan 15
23:11:01 2014
@@ -478,6 +478,9 @@ Trunk (Unreleased)
HDFS-5726. Fix compilation error in AbstractINodeDiff for JDK7. (jing9)
+ HDFS-5768. Consolidate the serialization code in
DelegationTokenSecretManager
+ (Haohui Mai via brandonli)
+
Release 2.4.0 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java?rev=1558598&r1=1558597&r2=1558598&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java
(original)
+++
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java
Wed Jan 15 23:11:01 2014
@@ -59,6 +59,7 @@ public class DelegationTokenSecretManage
.getLog(DelegationTokenSecretManager.class);
private final FSNamesystem namesystem;
+ private final SerializerCompat serializerCompat = new SerializerCompat();
public DelegationTokenSecretManager(long delegationKeyUpdateInterval,
long delegationTokenMaxLifetime, long delegationTokenRenewInterval,
@@ -150,24 +151,21 @@ public class DelegationTokenSecretManage
throw new IOException("No delegation token found for this identifier");
}
}
-
+
/**
* Load SecretManager state from fsimage.
*
* @param in input stream to read fsimage
* @throws IOException
*/
- public synchronized void loadSecretManagerState(DataInput in)
+ public synchronized void loadSecretManagerStateCompat(DataInput in)
throws IOException {
if (running) {
// a safety check
throw new IOException(
"Can't load state from image in a running SecretManager.");
}
- currentId = in.readInt();
- loadAllKeys(in);
- delegationTokenSequenceNumber = in.readInt();
- loadCurrentTokens(in);
+ serializerCompat.load(in);
}
/**
@@ -177,12 +175,9 @@ public class DelegationTokenSecretManage
* @param sdPath String storage directory path
* @throws IOException
*/
- public synchronized void saveSecretManagerState(DataOutputStream out,
+ public synchronized void saveSecretManagerStateCompat(DataOutputStream out,
String sdPath) throws IOException {
- out.writeInt(currentId);
- saveAllKeys(out, sdPath);
- out.writeInt(delegationTokenSequenceNumber);
- saveCurrentTokens(out, sdPath);
+ serializerCompat.save(out, sdPath);
}
/**
@@ -283,91 +278,6 @@ public class DelegationTokenSecretManage
}
/**
- * Private helper methods to save delegation keys and tokens in fsimage
- */
- private synchronized void saveCurrentTokens(DataOutputStream out,
- String sdPath) throws IOException {
- StartupProgress prog = NameNode.getStartupProgress();
- Step step = new Step(StepType.DELEGATION_TOKENS, sdPath);
- prog.beginStep(Phase.SAVING_CHECKPOINT, step);
- prog.setTotal(Phase.SAVING_CHECKPOINT, step, currentTokens.size());
- Counter counter = prog.getCounter(Phase.SAVING_CHECKPOINT, step);
- out.writeInt(currentTokens.size());
- Iterator<DelegationTokenIdentifier> iter = currentTokens.keySet()
- .iterator();
- while (iter.hasNext()) {
- DelegationTokenIdentifier id = iter.next();
- id.write(out);
- DelegationTokenInformation info = currentTokens.get(id);
- out.writeLong(info.getRenewDate());
- counter.increment();
- }
- prog.endStep(Phase.SAVING_CHECKPOINT, step);
- }
-
- /*
- * Save the current state of allKeys
- */
- private synchronized void saveAllKeys(DataOutputStream out, String sdPath)
- throws IOException {
- StartupProgress prog = NameNode.getStartupProgress();
- Step step = new Step(StepType.DELEGATION_KEYS, sdPath);
- prog.beginStep(Phase.SAVING_CHECKPOINT, step);
- prog.setTotal(Phase.SAVING_CHECKPOINT, step, currentTokens.size());
- Counter counter = prog.getCounter(Phase.SAVING_CHECKPOINT, step);
- out.writeInt(allKeys.size());
- Iterator<Integer> iter = allKeys.keySet().iterator();
- while (iter.hasNext()) {
- Integer key = iter.next();
- allKeys.get(key).write(out);
- counter.increment();
- }
- prog.endStep(Phase.SAVING_CHECKPOINT, step);
- }
-
- /**
- * Private helper methods to load Delegation tokens from fsimage
- */
- private synchronized void loadCurrentTokens(DataInput in)
- throws IOException {
- StartupProgress prog = NameNode.getStartupProgress();
- Step step = new Step(StepType.DELEGATION_TOKENS);
- prog.beginStep(Phase.LOADING_FSIMAGE, step);
- int numberOfTokens = in.readInt();
- prog.setTotal(Phase.LOADING_FSIMAGE, step, numberOfTokens);
- Counter counter = prog.getCounter(Phase.LOADING_FSIMAGE, step);
- for (int i = 0; i < numberOfTokens; i++) {
- DelegationTokenIdentifier id = new DelegationTokenIdentifier();
- id.readFields(in);
- long expiryTime = in.readLong();
- addPersistedDelegationToken(id, expiryTime);
- counter.increment();
- }
- prog.endStep(Phase.LOADING_FSIMAGE, step);
- }
-
- /**
- * Private helper method to load delegation keys from fsimage.
- * @param in
- * @throws IOException
- */
- private synchronized void loadAllKeys(DataInput in) throws IOException {
- StartupProgress prog = NameNode.getStartupProgress();
- Step step = new Step(StepType.DELEGATION_KEYS);
- prog.beginStep(Phase.LOADING_FSIMAGE, step);
- int numberOfKeys = in.readInt();
- prog.setTotal(Phase.LOADING_FSIMAGE, step, numberOfKeys);
- Counter counter = prog.getCounter(Phase.LOADING_FSIMAGE, step);
- for (int i = 0; i < numberOfKeys; i++) {
- DelegationKey value = new DelegationKey();
- value.readFields(in);
- addKey(value);
- counter.increment();
- }
- prog.endStep(Phase.LOADING_FSIMAGE, step);
- }
-
- /**
* Call namesystem to update editlogs for new master key.
*/
@Override //AbstractDelegationTokenManager
@@ -420,4 +330,105 @@ public class DelegationTokenSecretManage
c.addToken(new Text(ugi.getShortUserName()), token);
return c;
}
+
+ private final class SerializerCompat {
+ private void load(DataInput in) throws IOException {
+ currentId = in.readInt();
+ loadAllKeys(in);
+ delegationTokenSequenceNumber = in.readInt();
+ loadCurrentTokens(in);
+ }
+
+ private void save(DataOutputStream out, String sdPath) throws IOException {
+ out.writeInt(currentId);
+ saveAllKeys(out, sdPath);
+ out.writeInt(delegationTokenSequenceNumber);
+ saveCurrentTokens(out, sdPath);
+ }
+
+ /**
+ * Private helper methods to save delegation keys and tokens in fsimage
+ */
+ private synchronized void saveCurrentTokens(DataOutputStream out,
+ String sdPath) throws IOException {
+ StartupProgress prog = NameNode.getStartupProgress();
+ Step step = new Step(StepType.DELEGATION_TOKENS, sdPath);
+ prog.beginStep(Phase.SAVING_CHECKPOINT, step);
+ prog.setTotal(Phase.SAVING_CHECKPOINT, step, currentTokens.size());
+ Counter counter = prog.getCounter(Phase.SAVING_CHECKPOINT, step);
+ out.writeInt(currentTokens.size());
+ Iterator<DelegationTokenIdentifier> iter = currentTokens.keySet()
+ .iterator();
+ while (iter.hasNext()) {
+ DelegationTokenIdentifier id = iter.next();
+ id.write(out);
+ DelegationTokenInformation info = currentTokens.get(id);
+ out.writeLong(info.getRenewDate());
+ counter.increment();
+ }
+ prog.endStep(Phase.SAVING_CHECKPOINT, step);
+ }
+
+ /*
+ * Save the current state of allKeys
+ */
+ private synchronized void saveAllKeys(DataOutputStream out, String sdPath)
+ throws IOException {
+ StartupProgress prog = NameNode.getStartupProgress();
+ Step step = new Step(StepType.DELEGATION_KEYS, sdPath);
+ prog.beginStep(Phase.SAVING_CHECKPOINT, step);
+ prog.setTotal(Phase.SAVING_CHECKPOINT, step, currentTokens.size());
+ Counter counter = prog.getCounter(Phase.SAVING_CHECKPOINT, step);
+ out.writeInt(allKeys.size());
+ Iterator<Integer> iter = allKeys.keySet().iterator();
+ while (iter.hasNext()) {
+ Integer key = iter.next();
+ allKeys.get(key).write(out);
+ counter.increment();
+ }
+ prog.endStep(Phase.SAVING_CHECKPOINT, step);
+ }
+
+ /**
+ * Private helper methods to load Delegation tokens from fsimage
+ */
+ private synchronized void loadCurrentTokens(DataInput in)
+ throws IOException {
+ StartupProgress prog = NameNode.getStartupProgress();
+ Step step = new Step(StepType.DELEGATION_TOKENS);
+ prog.beginStep(Phase.LOADING_FSIMAGE, step);
+ int numberOfTokens = in.readInt();
+ prog.setTotal(Phase.LOADING_FSIMAGE, step, numberOfTokens);
+ Counter counter = prog.getCounter(Phase.LOADING_FSIMAGE, step);
+ for (int i = 0; i < numberOfTokens; i++) {
+ DelegationTokenIdentifier id = new DelegationTokenIdentifier();
+ id.readFields(in);
+ long expiryTime = in.readLong();
+ addPersistedDelegationToken(id, expiryTime);
+ counter.increment();
+ }
+ prog.endStep(Phase.LOADING_FSIMAGE, step);
+ }
+
+ /**
+ * Private helper method to load delegation keys from fsimage.
+ * @param in
+ * @throws IOException
+ */
+ private synchronized void loadAllKeys(DataInput in) throws IOException {
+ StartupProgress prog = NameNode.getStartupProgress();
+ Step step = new Step(StepType.DELEGATION_KEYS);
+ prog.beginStep(Phase.LOADING_FSIMAGE, step);
+ int numberOfKeys = in.readInt();
+ prog.setTotal(Phase.LOADING_FSIMAGE, step, numberOfKeys);
+ Counter counter = prog.getCounter(Phase.LOADING_FSIMAGE, step);
+ for (int i = 0; i < numberOfKeys; i++) {
+ DelegationKey value = new DelegationKey();
+ value.readFields(in);
+ addKey(value);
+ counter.increment();
+ }
+ prog.endStep(Phase.LOADING_FSIMAGE, step);
+ }
+ }
}
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java?rev=1558598&r1=1558597&r2=1558598&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
(original)
+++
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
Wed Jan 15 23:11:01 2014
@@ -870,7 +870,7 @@ public class FSImageFormat {
//This must not happen if security is turned on.
return;
}
- namesystem.loadSecretManagerState(in);
+ namesystem.loadSecretManagerStateCompat(in);
}
private void loadCacheManagerState(DataInput in) throws IOException {
@@ -1032,7 +1032,7 @@ public class FSImageFormat {
sourceNamesystem.saveFilesUnderConstruction(out, snapshotUCMap);
context.checkCancelled();
- sourceNamesystem.saveSecretManagerState(out, sdPath);
+ sourceNamesystem.saveSecretManagerStateCompat(out, sdPath);
context.checkCancelled();
sourceNamesystem.getCacheManager().saveState(out, sdPath);
context.checkCancelled();
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1558598&r1=1558597&r2=1558598&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
(original)
+++
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
Wed Jan 15 23:11:01 2014
@@ -6250,16 +6250,16 @@ public class FSNamesystem implements Nam
* @param out save state of the secret manager
* @param sdPath String storage directory path
*/
- void saveSecretManagerState(DataOutputStream out, String sdPath)
+ void saveSecretManagerStateCompat(DataOutputStream out, String sdPath)
throws IOException {
- dtSecretManager.saveSecretManagerState(out, sdPath);
+ dtSecretManager.saveSecretManagerStateCompat(out, sdPath);
}
/**
* @param in load the state of secret manager from input stream
*/
- void loadSecretManagerState(DataInput in) throws IOException {
- dtSecretManager.loadSecretManagerState(in);
+ void loadSecretManagerStateCompat(DataInput in) throws IOException {
+ dtSecretManager.loadSecretManagerStateCompat(in);
}
/**
