Author: shv
Date: Sun Jan 13 01:13:29 2013
New Revision: 1432539
URL: http://svn.apache.org/viewvc?rev=1432539&view=rev
Log:
HDFS-1245. Plugable block id generation. Contributed by Konstantin Shvachko.
Added:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RandomBlockIdGenerator.java
(with props)
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1432539&r1=1432538&r2=1432539&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Sun Jan 13
01:13:29 2013
@@ -701,6 +701,8 @@ Release 2.0.3-alpha - Unreleased
HDFS-4274. BlockPoolSliceScanner does not close verification log during
shutdown. (Chris Nauroth via suresh)
+ HDFS-1245. Plugable block id generation. (shv)
+
BREAKDOWN OF HDFS-3077 SUBTASKS
HDFS-3077. Quorum-based protocol for reading and writing edit logs.
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java?rev=1432539&r1=1432538&r2=1432539&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
(original)
+++
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
Sun Jan 13 01:13:29 2013
@@ -59,6 +59,7 @@ import org.apache.hadoop.hdfs.server.pro
import org.apache.hadoop.hdfs.util.Canceler;
import org.apache.hadoop.hdfs.util.MD5FileUtils;
import org.apache.hadoop.io.MD5Hash;
+import org.apache.hadoop.util.IdGenerator;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
@@ -92,6 +93,7 @@ public class FSImage implements Closeabl
final private Configuration conf;
protected NNStorageRetentionManager archivalManager;
+ protected IdGenerator blockIdGenerator;
/**
* Construct an FSImage
@@ -137,6 +139,9 @@ public class FSImage implements Closeabl
Preconditions.checkState(fileCount == 1,
"FSImage.format should be called with an uninitialized namesystem, has
" +
fileCount + " files");
+ // BlockIdGenerator is defined during formatting
+ // currently there is only one BlockIdGenerator
+ blockIdGenerator = createBlockIdGenerator(fsn);
NamespaceInfo ns = NNStorage.newNamespaceInfo();
ns.clusterID = clusterId;
@@ -253,6 +258,7 @@ public class FSImage implements Closeabl
doRollback();
break;
case REGULAR:
+ default:
// just load the image
}
@@ -737,6 +743,9 @@ public class FSImage implements Closeabl
FSImageFormat.Loader loader = new FSImageFormat.Loader(
conf, target);
loader.load(curFile);
+ // BlockIdGenerator is determined after loading image
+ // currently there is only one BlockIdGenerator
+ blockIdGenerator = createBlockIdGenerator(target);
target.setBlockPoolId(this.getBlockPoolID());
// Check that the image digest we loaded matches up with what
@@ -1165,4 +1174,12 @@ public class FSImage implements Closeabl
public synchronized long getMostRecentCheckpointTxId() {
return storage.getMostRecentCheckpointTxId();
}
+
+ public long getUniqueBlockId() {
+ return blockIdGenerator.nextValue();
+ }
+
+ public IdGenerator createBlockIdGenerator(FSNamesystem fsn) {
+ return new RandomBlockIdGenerator(fsn);
+ }
}
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1432539&r1=1432538&r2=1432539&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
(original)
+++
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
Sun Jan 13 01:13:29 2013
@@ -79,7 +79,6 @@ import java.io.DataOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
-import java.io.FileWriter;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
@@ -2539,10 +2538,7 @@ public class FSNamesystem implements Nam
private Block allocateBlock(String src, INodesInPath inodesInPath,
DatanodeDescriptor targets[]) throws IOException {
assert hasWriteLock();
- Block b = new Block(DFSUtil.getRandom().nextLong(), 0, 0);
- while(isValidBlock(b)) {
- b.setBlockId(DFSUtil.getRandom().nextLong());
- }
+ Block b = new Block(getFSImage().getUniqueBlockId(), 0, 0);
// Increment the generation stamp for every new block.
b.setGenerationStamp(nextGenerationStamp());
b = dir.addBlock(src, inodesInPath, b, targets);
@@ -4554,13 +4550,6 @@ public class FSNamesystem implements Nam
}
}
- /**
- * Returns whether the given block is one pointed-to by a file.
- */
- private boolean isValidBlock(Block b) {
- return (blockManager.getBlockCollection(b) != null);
- }
-
PermissionStatus createFsOwnerPermissions(FsPermission permission) {
return new PermissionStatus(fsOwner.getShortUserName(), supergroup,
permission);
}
Added:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RandomBlockIdGenerator.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RandomBlockIdGenerator.java?rev=1432539&view=auto
==============================================================================
---
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RandomBlockIdGenerator.java
(added)
+++
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RandomBlockIdGenerator.java
Sun Jan 13 01:13:29 2013
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
+import org.apache.hadoop.util.IdGenerator;
+
+/**
+ * Generator of random block IDs.
+ */
[email protected]
+public class RandomBlockIdGenerator implements IdGenerator {
+ private final BlockManager blockManager;
+
+ RandomBlockIdGenerator(FSNamesystem namesystem) {
+ this.blockManager = namesystem.getBlockManager();
+ }
+
+ @Override // NumberGenerator
+ public long nextValue() {
+ Block b = new Block(DFSUtil.getRandom().nextLong(), 0, 0);
+ while(isValidBlock(b)) {
+ b.setBlockId(DFSUtil.getRandom().nextLong());
+ }
+ return b.getBlockId();
+ }
+
+ /**
+ * Returns whether the given block is one pointed-to by a file.
+ */
+ private boolean isValidBlock(Block b) {
+ return (blockManager.getBlockCollection(b) != null);
+ }
+}
Propchange:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RandomBlockIdGenerator.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
