Author: szetszwo
Date: Thu Jan 12 03:42:12 2012
New Revision: 1230388
URL: http://svn.apache.org/viewvc?rev=1230388&view=rev
Log:
Merge r1210719 and r1210746 from trunk for HDFS-2618.
Added:
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolPB.java
- copied unchanged from r1210719,
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolPB.java
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
- copied unchanged from r1210719,
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java
- copied, changed from r1210719,
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java
Modified:
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/
(props changed)
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
(props changed)
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/JournalProtocolTranslatorPB.java
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockKey.java
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/NamenodeProtocol.proto
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
Propchange:
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Jan 12 03:42:12 2012
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:1161777,1161781,1161992,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163081,1163490,1163768,1164255,1164301,1164339,1165826,1166402,1166466,1167383,1167662,1170085,1170379,1170459,1170996,1171136,1171297,1171379,1171611,1171711,1172916,1173402,1173468,1173488,1175113,1176178,1176550,1176719,1176729,1176733,1177100,1177161,1177487,1177531,1177757,1177859,1177864,1177905,1179169,1179856,1179861,1180757,1183081,1183098,1183175,1183554,1186508,1186896,1187140,1187505,1188282,1188286,1188300,1188436,1188487,1189028,1189355,1189360,1189546,1189901,1189932,1189982,1190077,1190127,1190620,1190708,1195575,1195656,1195731,1195754,1196113,1196129,1196171,1197329,1198903,1199396,1200731,1204114,1204117,1204122,1204124,1204129,1204131,1204366,1204370,1204376,1204388,1204544,1204707,1204709,1204825,1205146,1205260,1205626,1206178,1206786,1206830,1207585,1207694,1208140,1208153,1208313,1210657,1212021,
1212062,1212073,1212084,1212299,1213537,1213586,1213592-1213593,1213954,1214027,1214046,1220510,1221106,1221348,1226211,1227091,1227423,1229897
+/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:1161777,1161781,1161992,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163081,1163490,1163768,1164255,1164301,1164339,1165826,1166402,1166466,1167383,1167662,1170085,1170379,1170459,1170996,1171136,1171297,1171379,1171611,1171711,1172916,1173402,1173468,1173488,1175113,1176178,1176550,1176719,1176729,1176733,1177100,1177161,1177487,1177531,1177757,1177859,1177864,1177905,1179169,1179856,1179861,1180757,1183081,1183098,1183175,1183554,1186508,1186896,1187140,1187505,1188282,1188286,1188300,1188436,1188487,1189028,1189355,1189360,1189546,1189901,1189932,1189982,1190077,1190127,1190620,1190708,1195575,1195656,1195731,1195754,1196113,1196129,1196171,1197329,1198903,1199396,1200731,1204114,1204117,1204122,1204124,1204129,1204131,1204366,1204370,1204376,1204388,1204544,1204707,1204709,1204825,1205146,1205260,1205626,1206178,1206786,1206830,1207585,1207694,1208140,1208153,1208313,1210657,1210719,
1210746,1212021,1212062,1212073,1212084,1212299,1213537,1213586,1213592-1213593,1213954,1214027,1214046,1220510,1221106,1221348,1226211,1227091,1227423,1229897
/hadoop/core/branches/branch-0.19/hdfs:713112
/hadoop/hdfs/branches/HDFS-1052:987665-1095512
/hadoop/hdfs/branches/HDFS-265:796829-820463
Modified:
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1230388&r1=1230387&r2=1230388&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Thu Jan 12 03:42:12 2012
@@ -17,6 +17,8 @@ Release 0.23-PB - Unreleased
HDFS-2581. Implement protobuf service for JournalProtocol. (suresh)
+ HDFS-2618. Implement protobuf service for NamenodeProtocol. (suresh)
+
IMPROVEMENTS
HDFS-2018. Move all journal stream management code into one place.
Propchange:
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Jan 12 03:42:12 2012
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:1161777,1161781,1161992,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163081,1163490,1163768,1164255,1164301,1164339,1165826,1166402,1166466,1167383,1167662,1170085,1170379,1170459,1170996,1171136,1171297,1171379,1171611,1171711,1172916,1173402,1173468,1173488,1175113,1176178,1176550,1176719,1176729,1176733,1177100,1177161,1177487,1177531,1177757,1177859,1177864,1177905,1179169,1179856,1179861,1180757,1183081,1183098,1183175,1183554,1186508,1186896,1187140,1187505,1188282,1188286,1188300,1188436,1188487,1189028,1189355,1189360,1189546,1189901,1189932,1189982,1190077,1190127,1190620,1190708,1195575,1195656,1195731,1195754,1196113,1196129,1196171,1197329,1198903,1199396,1200731,1204114,1204117,1204122,1204124,1204129,1204131,1204366,1204370,1204376,1204388,1204544,1204707,1205146,1205260,1206786,1206830,1207694,1208140,1208153,1208313,1210657,1212021,1212062,1212073,1212084,12
12299,1213537,1213586,1213592-1213593,1213954,1214027,1214046,1220510,1221106,1221348,1226211,1227091,1227423,1229897
+/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:1161777,1161781,1161992,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163081,1163490,1163768,1164255,1164301,1164339,1165826,1166402,1166466,1167383,1167662,1170085,1170379,1170459,1170996,1171136,1171297,1171379,1171611,1171711,1172916,1173402,1173468,1173488,1175113,1176178,1176550,1176719,1176729,1176733,1177100,1177161,1177487,1177531,1177757,1177859,1177864,1177905,1179169,1179856,1179861,1180757,1183081,1183098,1183175,1183554,1186508,1186896,1187140,1187505,1188282,1188286,1188300,1188436,1188487,1189028,1189355,1189360,1189546,1189901,1189932,1189982,1190077,1190127,1190620,1190708,1195575,1195656,1195731,1195754,1196113,1196129,1196171,1197329,1198903,1199396,1200731,1204114,1204117,1204122,1204124,1204129,1204131,1204366,1204370,1204376,1204388,1204544,1204707,1205146,1205260,1206786,1206830,1207694,1208140,1208153,1208313,1210657,1210719,1210746,1212021,1212062,12
12073,1212084,1212299,1213537,1213586,1213592-1213593,1213954,1214027,1214046,1220510,1221106,1221348,1226211,1227091,1227423,1229897
/hadoop/core/branches/branch-0.19/hdfs/src/java:713112
/hadoop/core/trunk/src/hdfs:776175-785643,785929-786278
/hadoop/hdfs/branches/HDFS-1052/src/java:987665-1095512
Modified:
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/JournalProtocolTranslatorPB.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/JournalProtocolTranslatorPB.java?rev=1230388&r1=1230387&r2=1230388&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/JournalProtocolTranslatorPB.java
(original)
+++
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/JournalProtocolTranslatorPB.java
Thu Jan 12 03:42:12 2012
@@ -53,7 +53,7 @@ public class JournalProtocolTranslatorPB
Configuration conf) throws IOException {
RPC.setProtocolEngine(conf, JournalProtocolPB.class,
ProtobufRpcEngine.class);
rpcProxy = RPC.getProxy(JournalProtocolPB.class,
- JournalProtocol.versionID, nameNodeAddr, conf);
+ RPC.getProtocolVersion(JournalProtocolPB.class), nameNodeAddr, conf);
}
@Override
@@ -64,7 +64,7 @@ public class JournalProtocolTranslatorPB
@Override
public long getProtocolVersion(String protocolName, long clientVersion)
throws IOException {
- return 0;
+ return rpcProxy.getProtocolVersion(protocolName, clientVersion);
}
@Override
Copied:
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java
(from r1210719,
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java)
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java?p2=hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java&p1=hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java&r1=1210719&r2=1230388&rev=1230388&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java
(original)
+++
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java
Thu Jan 12 03:42:12 2012
@@ -206,7 +206,7 @@ public class NamenodeProtocolTranslatorP
String msg) throws IOException {
ErrorReportRequestProto req = ErrorReportRequestProto.newBuilder()
.setErrorCode(errorCode).setMsg(msg)
- .setRegistartion(PBHelper.convert(registration)).build();
+ .setRegistration(PBHelper.convert(registration)).build();
try {
rpcProxy.errorReport(NULL_CONTROLLER, req);
} catch (ServiceException e) {
@@ -266,4 +266,4 @@ public class NamenodeProtocolTranslatorP
throw ProtobufHelper.getRemoteException(e);
}
}
-}
\ No newline at end of file
+}
Modified:
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java?rev=1230388&r1=1230387&r2=1230388&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
(original)
+++
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
Thu Jan 12 03:42:12 2012
@@ -17,28 +17,55 @@
*/
package org.apache.hadoop.hdfs.protocolPB;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.DatanodeID;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockKeyProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto;
+import
org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockWithLocationsProto;
+import
org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlocksWithLocationsProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.CheckpointCommandProto;
+import
org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.CheckpointSignatureProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeIDProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ExportedBlockKeysProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.NamenodeCommandProto;
import
org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.NamenodeRegistrationProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.NamespaceInfoProto;
+import
org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.RemoteEditLogManifestProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.RemoteEditLogProto;
import
org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.NamenodeRegistrationProto.NamenodeRoleProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageInfoProto;
+import org.apache.hadoop.hdfs.security.token.block.BlockKey;
+import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
import org.apache.hadoop.hdfs.server.common.StorageInfo;
+import org.apache.hadoop.hdfs.server.namenode.CheckpointSignature;
+import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations;
+import org.apache.hadoop.hdfs.server.protocol.CheckpointCommand;
+import org.apache.hadoop.hdfs.server.protocol.NamenodeCommand;
import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration;
+import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
+import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog;
+import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
+import
org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations.BlockWithLocations;
import com.google.protobuf.ByteString;
/**
- * Utilities for converting protobuf classes to and from
- * implementation classes.
+ * Utilities for converting protobuf classes to and from implementation
classes.
*/
class PBHelper {
private PBHelper() {
/** Hidden constructor */
}
-
+
public static ByteString getByteString(byte[] bytes) {
return ByteString.copyFrom(bytes);
}
-
+
public static NamenodeRole convert(NamenodeRoleProto role) {
switch (role) {
case NAMENODE:
@@ -50,7 +77,7 @@ class PBHelper {
}
return null;
}
-
+
public static NamenodeRoleProto convert(NamenodeRole role) {
switch (role) {
case NAMENODE:
@@ -62,31 +89,196 @@ class PBHelper {
}
return null;
}
-
+
public static StorageInfoProto convert(StorageInfo info) {
return StorageInfoProto.newBuilder().setClusterID(info.getClusterID())
- .setCTime(info.getCTime())
- .setLayoutVersion(info.getLayoutVersion())
- .setNamespceID(info.getNamespaceID())
- .build();
+ .setCTime(info.getCTime()).setLayoutVersion(info.getLayoutVersion())
+ .setNamespceID(info.getNamespaceID()).build();
}
-
+
public static StorageInfo convert(StorageInfoProto info) {
return new StorageInfo(info.getLayoutVersion(), info.getNamespceID(),
info.getClusterID(), info.getCTime());
}
-
-
+
public static NamenodeRegistrationProto convert(NamenodeRegistration reg) {
return NamenodeRegistrationProto.newBuilder()
- .setHttpAddress(reg.getHttpAddress())
- .setRole(convert(reg.getRole()))
+ .setHttpAddress(reg.getHttpAddress()).setRole(convert(reg.getRole()))
.setRpcAddress(reg.getAddress())
.setStorageInfo(convert((StorageInfo) reg)).build();
}
-
+
public static NamenodeRegistration convert(NamenodeRegistrationProto reg) {
return new NamenodeRegistration(reg.getRpcAddress(), reg.getHttpAddress(),
convert(reg.getStorageInfo()), convert(reg.getRole()));
}
-}
+
+ public static DatanodeID convert(DatanodeIDProto dn) {
+ return new DatanodeID(dn.getName(), dn.getStorageID(), dn.getInfoPort(),
+ dn.getIpcPort());
+ }
+
+ public static DatanodeIDProto convert(DatanodeID dn) {
+ return DatanodeIDProto.newBuilder().setName(dn.getName())
+ .setInfoPort(dn.getInfoPort()).setIpcPort(dn.getIpcPort())
+ .setStorageID(dn.getStorageID()).build();
+ }
+
+ public static BlockProto convert(Block b) {
+ return BlockProto.newBuilder().setBlockId(b.getBlockId())
+ .setGenStamp(b.getGenerationStamp()).setNumBytes(b.getNumBytes())
+ .build();
+ }
+
+ public static Block convert(BlockProto b) {
+ return new Block(b.getBlockId(), b.getGenStamp(), b.getNumBytes());
+ }
+
+ public static BlockWithLocationsProto convert(BlockWithLocations blk) {
+ return BlockWithLocationsProto.newBuilder()
+ .setBlock(convert(blk.getBlock()))
+ .addAllDatanodeIDs(Arrays.asList(blk.getDatanodes())).build();
+ }
+
+ public static BlockWithLocations convert(BlockWithLocationsProto b) {
+ return new BlockWithLocations(convert(b.getBlock()), b.getDatanodeIDsList()
+ .toArray(new String[0]));
+ }
+
+ public static BlocksWithLocationsProto convert(BlocksWithLocations blks) {
+ BlocksWithLocationsProto.Builder builder = BlocksWithLocationsProto
+ .newBuilder();
+ for (BlockWithLocations b : blks.getBlocks()) {
+ builder.addBlocks(convert(b));
+ }
+ return builder.build();
+ }
+
+ public static BlocksWithLocations convert(BlocksWithLocationsProto blocks) {
+ return new BlocksWithLocations(convert(blocks.getBlocksList()));
+ }
+
+ public static BlockKeyProto convert(BlockKey key) {
+ byte[] encodedKey = key.getEncodedKey();
+ ByteString keyBytes = ByteString.copyFrom(encodedKey == null ? new byte[0]
+ : encodedKey);
+ return BlockKeyProto.newBuilder().setKeyId(key.getKeyId())
+ .setKeyBytes(keyBytes).setExpiryDate(key.getExpiryDate()).build();
+ }
+
+ public static BlockKey convert(BlockKeyProto k) {
+ return new BlockKey(k.getKeyId(), k.getExpiryDate(), k.getKeyBytes()
+ .toByteArray());
+ }
+
+ public static ExportedBlockKeysProto convert(ExportedBlockKeys keys) {
+ ExportedBlockKeysProto.Builder builder = ExportedBlockKeysProto
+ .newBuilder();
+ builder.setIsBlockTokenEnabled(keys.isBlockTokenEnabled())
+ .setKeyUpdateInterval(keys.getKeyUpdateInterval())
+ .setTokenLifeTime(keys.getTokenLifetime())
+ .setCurrentKey(convert(keys.getCurrentKey()));
+ for (BlockKey k : keys.getAllKeys()) {
+ builder.addAllKeys(convert(k));
+ }
+ return builder.build();
+ }
+
+ public static ExportedBlockKeys convert(ExportedBlockKeysProto keys) {
+ return new ExportedBlockKeys(keys.getIsBlockTokenEnabled(),
+ keys.getKeyUpdateInterval(), keys.getTokenLifeTime(),
+ convert(keys.getCurrentKey()),
convertBlockKeys(keys.getAllKeysList()));
+ }
+
+ public static CheckpointSignatureProto convert(CheckpointSignature s) {
+ return CheckpointSignatureProto.newBuilder()
+ .setBlockPoolId(s.getBlockpoolID())
+ .setCurSegmentTxId(s.getCurSegmentTxId())
+ .setMostRecentCheckpointTxId(s.getMostRecentCheckpointTxId())
+ .setStorageInfo(PBHelper.convert((StorageInfo) s)).build();
+ }
+
+ public static CheckpointSignature convert(CheckpointSignatureProto s) {
+ return new CheckpointSignature(PBHelper.convert(s.getStorageInfo()),
+ s.getBlockPoolId(), s.getMostRecentCheckpointTxId(),
+ s.getCurSegmentTxId());
+ }
+
+ public static RemoteEditLogProto convert(RemoteEditLog log) {
+ return RemoteEditLogProto.newBuilder().setEndTxId(log.getEndTxId())
+ .setStartTxId(log.getStartTxId()).build();
+ }
+
+ public static RemoteEditLog convert(RemoteEditLogProto l) {
+ return new RemoteEditLog(l.getStartTxId(), l.getEndTxId());
+ }
+
+ public static RemoteEditLogManifestProto convert(
+ RemoteEditLogManifest manifest) {
+ RemoteEditLogManifestProto.Builder builder = RemoteEditLogManifestProto
+ .newBuilder();
+ for (RemoteEditLog log : manifest.getLogs()) {
+ builder.addLogs(convert(log));
+ }
+ return builder.build();
+ }
+
+ public static RemoteEditLogManifest convert(
+ RemoteEditLogManifestProto manifest) {
+ List<RemoteEditLog> logs = new ArrayList<RemoteEditLog>(manifest
+ .getLogsList().size());
+ for (RemoteEditLogProto l : manifest.getLogsList()) {
+ logs.add(convert(l));
+ }
+ return new RemoteEditLogManifest(logs);
+ }
+
+ public static CheckpointCommandProto convert(CheckpointCommand cmd) {
+ return CheckpointCommandProto.newBuilder()
+ .setSignature(convert(cmd.getSignature())).build();
+ }
+
+ public static NamenodeCommandProto convert(NamenodeCommand cmd) {
+ if (cmd instanceof CheckpointCommand) {
+ return NamenodeCommandProto.newBuilder().setAction(cmd.getAction())
+ .setType(NamenodeCommandProto.Type.NamenodeCommand)
+ .setCheckpointCmd(convert((CheckpointCommand) cmd)).build();
+ }
+ return
NamenodeCommandProto.newBuilder().setAction(cmd.getAction()).build();
+ }
+
+ public static BlockWithLocations[] convert(List<BlockWithLocationsProto> b) {
+ BlockWithLocations[] ret = new BlockWithLocations[b.size()];
+ int i = 0;
+ for (BlockWithLocationsProto entry : b) {
+ ret[i++] = convert(entry);
+ }
+ return ret;
+ }
+
+ public static BlockKey[] convertBlockKeys(List<BlockKeyProto> list) {
+ BlockKey[] ret = new BlockKey[list.size()];
+ int i = 0;
+ for (BlockKeyProto k : list) {
+ ret[i++] = convert(k);
+ }
+ return ret;
+ }
+
+ public static NamespaceInfo convert(NamespaceInfoProto info) {
+ StorageInfoProto storage = info.getStorageInfo();
+ return new NamespaceInfo(storage.getNamespceID(), storage.getClusterID(),
+ info.getBlockPoolID(), storage.getCTime(),
info.getDistUpgradeVersion());
+ }
+
+ public static NamenodeCommand convert(NamenodeCommandProto cmd) {
+ switch (cmd.getType()) {
+ case CheckPointCommand:
+ CheckpointCommandProto chkPt = cmd.getCheckpointCmd();
+ return new CheckpointCommand(PBHelper.convert(chkPt.getSignature()),
+ chkPt.getNeedToReturnImage());
+ default:
+ return new NamenodeCommand(cmd.getAction());
+ }
+ }
+}
\ No newline at end of file
Modified:
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockKey.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockKey.java?rev=1230388&r1=1230387&r2=1230388&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockKey.java
(original)
+++
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockKey.java
Thu Jan 12 03:42:12 2012
@@ -36,4 +36,8 @@ public class BlockKey extends Delegation
public BlockKey(int keyId, long expiryDate, SecretKey key) {
super(keyId, expiryDate, key);
}
+
+ public BlockKey(int keyId, long expiryDate, byte[] encodedKey) {
+ super(keyId, expiryDate, encodedKey);
+ }
}
Modified:
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/NamenodeProtocol.proto
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/NamenodeProtocol.proto?rev=1230388&r1=1230387&r2=1230388&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/NamenodeProtocol.proto
(original)
+++
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/NamenodeProtocol.proto
Thu Jan 12 03:42:12 2012
@@ -42,7 +42,7 @@ message GetBlocksRequestProto {
* blocks - List of returned blocks
*/
message GetBlocksResponseProto {
- required BlockWithLocationsProto blocks = 1; // List of blocks
+ required BlocksWithLocationsProto blocks = 1; // List of blocks
}
/**
@@ -85,12 +85,25 @@ message RollEditLogResponseProto {
}
/**
- * registartion - Namenode reporting the error
+ * void request
+ */
+message VersionRequestProto {
+}
+
+/**
+ * void request
+ */
+message VersionResponseProto {
+ required NamespaceInfoProto info = 1;
+}
+
+/**
+ * registration - Namenode reporting the error
* errorCode - error code indicating the error
* msg - Free text description of the error
*/
message ErrorReportRequestProto {
- required NamenodeRegistrationProto registartion = 1; // Registartion info
+ required NamenodeRegistrationProto registration = 1; // Registration info
required uint32 errorCode = 2; // Error code
required string msg = 3; // Error message
}
@@ -194,6 +207,11 @@ service NamenodeProtocolService {
rpc rollEditLog(RollEditLogRequestProto) returns(RollEditLogResponseProto);
/**
+ * Close the current editlog and open a new one for checkpointing purposes
+ */
+ rpc versionRequest(VersionRequestProto) returns(VersionResponseProto);
+
+ /**
* Report from a sub-ordinate namenode of an error to the active namenode.
* Active namenode may decide to unregister the reporting namenode
* depending on the error.
Modified:
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto?rev=1230388&r1=1230387&r2=1230388&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
(original)
+++
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
Thu Jan 12 03:42:12 2012
@@ -270,8 +270,8 @@ message BlockProto {
* Block and datanodes where is it located
*/
message BlockWithLocationsProto {
- required BlockProto block = 1; // Block
- repeated DatanodeIDProto datanodeIDs = 2; // Datanodes with replicas of the
block
+ required BlockProto block = 1; // Block
+ repeated string datanodeIDs = 2; // Datanodes with replicas of the block
}
/**
Modified:
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java?rev=1230388&r1=1230387&r2=1230388&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
(original)
+++
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
Thu Jan 12 03:42:12 2012
@@ -19,12 +19,35 @@ package org.apache.hadoop.hdfs.protocolP
import static junit.framework.Assert.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.DatanodeID;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockKeyProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto;
+import
org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockWithLocationsProto;
+import
org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlocksWithLocationsProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.CheckpointCommandProto;
+import
org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.CheckpointSignatureProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeIDProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ExportedBlockKeysProto;
import
org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.NamenodeRegistrationProto;
import
org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.NamenodeRegistrationProto.NamenodeRoleProto;
+import
org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.RemoteEditLogManifestProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.RemoteEditLogProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageInfoProto;
+import org.apache.hadoop.hdfs.security.token.block.BlockKey;
+import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
import org.apache.hadoop.hdfs.server.common.StorageInfo;
+import org.apache.hadoop.hdfs.server.namenode.CheckpointSignature;
+import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations;
+import
org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations.BlockWithLocations;
import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration;
+import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog;
+import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
import org.junit.Test;
/**
@@ -46,10 +69,14 @@ public class TestPBHelper {
assertEquals(NamenodeRole.NAMENODE,
PBHelper.convert(NamenodeRoleProto.NAMENODE));
}
-
+
+ private static StorageInfo getStorageInfo() {
+ return new StorageInfo(1, 2, "cid", 3);
+ }
+
@Test
public void testConvertStoragInfo() {
- StorageInfo info = new StorageInfo(1, 2, "cid", 3);
+ StorageInfo info = getStorageInfo();
StorageInfoProto infoProto = PBHelper.convert(info);
StorageInfo info2 = PBHelper.convert(infoProto);
assertEquals(info.getClusterID(), info2.getClusterID());
@@ -57,10 +84,10 @@ public class TestPBHelper {
assertEquals(info.getLayoutVersion(), info2.getLayoutVersion());
assertEquals(info.getNamespaceID(), info2.getNamespaceID());
}
-
+
@Test
public void testConvertNamenodeRegistration() {
- StorageInfo info = new StorageInfo(1, 2, "cid", 3);
+ StorageInfo info = getStorageInfo();
NamenodeRegistration reg = new NamenodeRegistration("address:999",
"http:1000", info, NamenodeRole.NAMENODE);
NamenodeRegistrationProto regProto = PBHelper.convert(reg);
@@ -74,6 +101,144 @@ public class TestPBHelper {
assertEquals(reg.getRegistrationID(), reg2.getRegistrationID());
assertEquals(reg.getRole(), reg2.getRole());
assertEquals(reg.getVersion(), reg2.getVersion());
+
+ }
+
+ @Test
+ public void testConvertDatanodeID() {
+ DatanodeID dn = new DatanodeID("node", "sid", 1, 2);
+ DatanodeIDProto dnProto = PBHelper.convert(dn);
+ DatanodeID dn2 = PBHelper.convert(dnProto);
+ assertEquals(dn.getHost(), dn2.getHost());
+ assertEquals(dn.getInfoPort(), dn2.getInfoPort());
+ assertEquals(dn.getIpcPort(), dn2.getIpcPort());
+ assertEquals(dn.getName(), dn2.getName());
+ assertEquals(dn.getPort(), dn2.getPort());
+ assertEquals(dn.getStorageID(), dn2.getStorageID());
+ }
+
+ @Test
+ public void testConvertBlock() {
+ Block b = new Block(1, 100, 3);
+ BlockProto bProto = PBHelper.convert(b);
+ Block b2 = PBHelper.convert(bProto);
+ assertEquals(b, b2);
+ }
+
+ private static BlockWithLocations getBlockWithLocations(int bid) {
+ return new BlockWithLocations(new Block(bid, 0, 1), new String[] { "dn1",
+ "dn2", "dn3" });
+ }
+
+ private void compare(BlockWithLocations locs1, BlockWithLocations locs2) {
+ assertEquals(locs1.getBlock(), locs2.getBlock());
+ assertTrue(Arrays.equals(locs1.getDatanodes(), locs2.getDatanodes()));
+ }
+
+ @Test
+ public void testConvertBlockWithLocations() {
+ BlockWithLocations locs = getBlockWithLocations(1);
+ BlockWithLocationsProto locsProto = PBHelper.convert(locs);
+ BlockWithLocations locs2 = PBHelper.convert(locsProto);
+ compare(locs, locs2);
+ }
+
+ @Test
+ public void testConvertBlocksWithLocations() {
+ BlockWithLocations[] list = new BlockWithLocations[] {
+ getBlockWithLocations(1), getBlockWithLocations(2) };
+ BlocksWithLocations locs = new BlocksWithLocations(list);
+ BlocksWithLocationsProto locsProto = PBHelper.convert(locs);
+ BlocksWithLocations locs2 = PBHelper.convert(locsProto);
+ BlockWithLocations[] blocks = locs.getBlocks();
+ BlockWithLocations[] blocks2 = locs2.getBlocks();
+ assertEquals(blocks.length, blocks2.length);
+ for (int i = 0; i < blocks.length; i++) {
+ compare(blocks[i], blocks2[i]);
+ }
+ }
+
+ private static BlockKey getBlockKey(int keyId) {
+ return new BlockKey(keyId, 10, "encodedKey".getBytes());
+ }
+
+ private void compare(BlockKey k1, BlockKey k2) {
+ assertEquals(k1.getExpiryDate(), k2.getExpiryDate());
+ assertEquals(k1.getKeyId(), k2.getKeyId());
+ assertTrue(Arrays.equals(k1.getEncodedKey(), k2.getEncodedKey()));
+
+ }
+
+ @Test
+ public void testConvertBlockKey() {
+ BlockKey key = getBlockKey(1);
+ BlockKeyProto keyProto = PBHelper.convert(key);
+ BlockKey key1 = PBHelper.convert(keyProto);
+ compare(key, key1);
+ }
+
+ @Test
+ public void testConvertExportedBlockKeys() {
+ BlockKey[] keys = new BlockKey[] { getBlockKey(2), getBlockKey(3) };
+ ExportedBlockKeys expKeys = new ExportedBlockKeys(true, 9, 10,
+ getBlockKey(1), keys);
+ ExportedBlockKeysProto expKeysProto = PBHelper.convert(expKeys);
+ ExportedBlockKeys expKeys1 = PBHelper.convert(expKeysProto);
+
+ BlockKey[] allKeys = expKeys.getAllKeys();
+ BlockKey[] allKeys1 = expKeys1.getAllKeys();
+ assertEquals(allKeys.length, allKeys1.length);
+ for (int i = 0; i < allKeys.length; i++) {
+ compare(allKeys[i], allKeys1[i]);
+ }
+ compare(expKeys.getCurrentKey(), expKeys1.getCurrentKey());
+ assertEquals(expKeys.getKeyUpdateInterval(),
+ expKeys1.getKeyUpdateInterval());
+ assertEquals(expKeys.getTokenLifetime(), expKeys1.getTokenLifetime());
+ }
+
+ @Test
+ public void testConvertCheckpointSignature() {
+ CheckpointSignature s = new CheckpointSignature(getStorageInfo(), "bpid",
+ 100, 1);
+ CheckpointSignatureProto sProto = PBHelper.convert(s);
+ CheckpointSignature s1 = PBHelper.convert(sProto);
+ assertEquals(s.getBlockpoolID(), s1.getBlockpoolID());
+ assertEquals(s.getClusterID(), s1.getClusterID());
+ assertEquals(s.getCTime(), s1.getCTime());
+ assertEquals(s.getCurSegmentTxId(), s1.getCurSegmentTxId());
+ assertEquals(s.getLayoutVersion(), s1.getLayoutVersion());
+ assertEquals(s.getMostRecentCheckpointTxId(),
+ s1.getMostRecentCheckpointTxId());
+ assertEquals(s.getNamespaceID(), s1.getNamespaceID());
+ }
+
+ private static void compare(RemoteEditLog l1, RemoteEditLog l2) {
+ assertEquals(l1.getEndTxId(), l2.getEndTxId());
+ assertEquals(l1.getStartTxId(), l2.getStartTxId());
+ }
+
+ @Test
+ public void testConvertRemoteEditLog() {
+ RemoteEditLog l = new RemoteEditLog(1, 100);
+ RemoteEditLogProto lProto = PBHelper.convert(l);
+ RemoteEditLog l1 = PBHelper.convert(lProto);
+ compare(l, l1);
+ }
+
+ @Test
+ public void testConvertRemoteEditLogManifest() {
+ List<RemoteEditLog> logs = new ArrayList<RemoteEditLog>();
+ logs.add(new RemoteEditLog(1, 10));
+ logs.add(new RemoteEditLog(11, 20));
+ RemoteEditLogManifest m = new RemoteEditLogManifest(logs);
+ RemoteEditLogManifestProto mProto = PBHelper.convert(m);
+ RemoteEditLogManifest m1 = PBHelper.convert(mProto);
+ List<RemoteEditLog> logs1 = m1.getLogs();
+ assertEquals(logs.size(), logs1.size());
+ for (int i = 0; i < logs.size(); i++) {
+ compare(logs.get(i), logs1.get(i));
+ }
}
}
