Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java?rev=1214518&r1=1214517&r2=1214518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java Wed Dec 14 22:47:41 2011 @@ -62,11 +62,15 @@ import org.apache.hadoop.hdfs.protocol.U import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.hadoop.hdfs.protocol.HdfsConstants.UpgradeAction; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ClientNamenodeProtocol; import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.NamenodeProtocolService; +import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.DatanodeProtocolService; +import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolPB; +import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolServerSideTranslatorPB; import org.apache.hadoop.hdfs.protocolPB.NamenodeProtocolPB; import org.apache.hadoop.hdfs.protocolPB.NamenodeProtocolServerSideTranslatorPB; -import org.apache.hadoop.hdfs.protocolR23Compatible.ClientNamenodeWireProtocol; -import org.apache.hadoop.hdfs.protocolR23Compatible.ClientNamenodeProtocolServerSideTranslatorR23; +import org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolPB; +import org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB; import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.hdfs.server.common.IncorrectVersionException; @@ -92,6 +96,7 @@ import org.apache.hadoop.hdfs.server.pro import org.apache.hadoop.hdfs.server.protocol.UpgradeCommand; import org.apache.hadoop.io.EnumSetWritable; import org.apache.hadoop.io.Text; +import org.apache.hadoop.ipc.ProtobufRpcEngine; import org.apache.hadoop.ipc.ProtocolSignature; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ipc.Server; @@ -144,14 +149,23 @@ class NameNodeRpcServer implements Namen conf.getInt(DFS_DATANODE_HANDLER_COUNT_KEY, DFS_DATANODE_HANDLER_COUNT_DEFAULT); InetSocketAddress socAddr = nn.getRpcServerAddress(conf); - ClientNamenodeProtocolServerSideTranslatorR23 - clientProtocolServerTranslator = - new ClientNamenodeProtocolServerSideTranslatorR23(this); + RPC.setProtocolEngine(conf, ClientNamenodeProtocolPB.class, + ProtobufRpcEngine.class); + ClientNamenodeProtocolServerSideTranslatorPB + clientProtocolServerTranslator = + new ClientNamenodeProtocolServerSideTranslatorPB(this); + BlockingService clientNNPbService = ClientNamenodeProtocol. + newReflectiveBlockingService(clientProtocolServerTranslator); + DatanodeProtocolServerSideTranslatorPB dnProtoPbTranslator = + new DatanodeProtocolServerSideTranslatorPB(this); + BlockingService dnProtoPbService = DatanodeProtocolService + .newReflectiveBlockingService(dnProtoPbTranslator); + NamenodeProtocolServerSideTranslatorPB namenodeProtocolXlator = new NamenodeProtocolServerSideTranslatorPB(this); - BlockingService service = NamenodeProtocolService - .newReflectiveBlockingService(namenodeProtocolXlator); + BlockingService NNPbService = NamenodeProtocolService + .newReflectiveBlockingService(namenodeProtocolXlator); InetSocketAddress dnSocketAddr = nn.getServiceRpcServerAddress(conf); if (dnSocketAddr != null) { @@ -160,14 +174,12 @@ class NameNodeRpcServer implements Namen DFS_NAMENODE_SERVICE_HANDLER_COUNT_DEFAULT); // Add all the RPC protocols that the namenode implements this.serviceRpcServer = - RPC.getServer(org.apache.hadoop.hdfs.protocolR23Compatible. - ClientNamenodeWireProtocol.class, clientProtocolServerTranslator, + RPC.getServer(org.apache.hadoop.hdfs.protocolPB. + ClientNamenodeProtocolPB.class, clientNNPbService, dnSocketAddr.getHostName(), dnSocketAddr.getPort(), serviceHandlerCount, false, conf, namesystem.getDelegationTokenSecretManager()); this.serviceRpcServer.addProtocol(RpcKind.RPC_WRITABLE, - DatanodeProtocol.class, this); - this.serviceRpcServer.addProtocol(RpcKind.RPC_WRITABLE, RefreshAuthorizationPolicyProtocol.class, this); this.serviceRpcServer.addProtocol(RpcKind.RPC_WRITABLE, RefreshUserMappingsProtocol.class, this); @@ -175,7 +187,9 @@ class NameNodeRpcServer implements Namen GetUserMappingsProtocol.class, this); this.serviceRpcServer.addProtocol(RpcKind.RPC_WRITABLE, HAServiceProtocol.class, this); - DFSUtil.addPBProtocol(conf, NamenodeProtocolPB.class, service, + DFSUtil.addPBProtocol(conf, NamenodeProtocolPB.class, NNPbService, + serviceRpcServer); + DFSUtil.addPBProtocol(conf, DatanodeProtocolPB.class, dnProtoPbService, serviceRpcServer); this.serviceRPCAddress = this.serviceRpcServer.getListenerAddress(); @@ -186,14 +200,11 @@ class NameNodeRpcServer implements Namen } // Add all the RPC protocols that the namenode implements this.clientRpcServer = RPC.getServer( - org.apache.hadoop.hdfs.protocolR23Compatible. - ClientNamenodeWireProtocol.class, - clientProtocolServerTranslator, socAddr.getHostName(), + org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolPB.class, + clientNNPbService, socAddr.getHostName(), socAddr.getPort(), handlerCount, false, conf, namesystem.getDelegationTokenSecretManager()); this.clientRpcServer.addProtocol(RpcKind.RPC_WRITABLE, - DatanodeProtocol.class, this); - this.clientRpcServer.addProtocol(RpcKind.RPC_WRITABLE, RefreshAuthorizationPolicyProtocol.class, this); this.clientRpcServer.addProtocol(RpcKind.RPC_WRITABLE, RefreshUserMappingsProtocol.class, this); @@ -201,7 +212,9 @@ class NameNodeRpcServer implements Namen GetUserMappingsProtocol.class, this); this.clientRpcServer.addProtocol(RpcKind.RPC_WRITABLE, HAServiceProtocol.class, this); - DFSUtil.addPBProtocol(conf, NamenodeProtocolPB.class, service, + DFSUtil.addPBProtocol(conf, NamenodeProtocolPB.class, NNPbService, + clientRpcServer); + DFSUtil.addPBProtocol(conf, DatanodeProtocolPB.class, dnProtoPbService, clientRpcServer); // set service-level authorization security policy @@ -261,7 +274,7 @@ class NameNodeRpcServer implements Namen long clientVersion) throws IOException { if (protocol.equals(ClientProtocol.class.getName())) { throw new IOException("Old Namenode Client protocol is not supported:" + - protocol + "Switch your clientside to " + ClientNamenodeWireProtocol.class); + protocol + "Switch your clientside to " + ClientNamenodeProtocol.class); } else if (protocol.equals(DatanodeProtocol.class.getName())){ return DatanodeProtocol.versionID; } else if (protocol.equals(NamenodeProtocol.class.getName())){
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java?rev=1214518&r1=1214517&r2=1214518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java Wed Dec 14 22:47:41 2011 @@ -76,6 +76,7 @@ import org.apache.hadoop.hdfs.web.resour import org.apache.hadoop.hdfs.web.resources.HttpOpParam; import org.apache.hadoop.hdfs.web.resources.LengthParam; import org.apache.hadoop.hdfs.web.resources.ModificationTimeParam; +import org.apache.hadoop.hdfs.web.resources.NamenodeRpcAddressParam; import org.apache.hadoop.hdfs.web.resources.OffsetParam; import org.apache.hadoop.hdfs.web.resources.OverwriteParam; import org.apache.hadoop.hdfs.web.resources.OwnerParam; @@ -198,6 +199,7 @@ public class NamenodeWebHdfsMethods { delegationQuery = "&" + new DelegationParam(t.encodeToUrlString()); } final String query = op.toQueryString() + delegationQuery + + "&" + new NamenodeRpcAddressParam(namenode) + Param.toSortedString("&", parameters); final String uripath = WebHdfsFileSystem.PATH_PREFIX + path; Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeCommand.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeCommand.java?rev=1214518&r1=1214517&r2=1214518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeCommand.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeCommand.java Wed Dec 14 22:47:41 2011 @@ -17,7 +17,6 @@ */ package org.apache.hadoop.hdfs.server.protocol; -import org.apache.avro.reflect.Union; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -25,13 +24,6 @@ import org.apache.hadoop.classification. * Base class for data-node command. * Issued by the name-node to notify data-nodes what should be done. */ - -// Declare subclasses for Avro's denormalized representation -@Union({Void.class, - RegisterCommand.class, FinalizeCommand.class, - BlockCommand.class, UpgradeCommand.class, - BlockRecoveryCommand.class, KeyUpdateCommand.class}) - @InterfaceAudience.Private @InterfaceStability.Evolving public abstract class DatanodeCommand extends ServerCommand { Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java?rev=1214518&r1=1214517&r2=1214518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java Wed Dec 14 22:47:41 2011 @@ -30,8 +30,6 @@ import org.apache.hadoop.hdfs.server.pro import org.apache.hadoop.ipc.VersionedProtocol; import org.apache.hadoop.security.KerberosInfo; -import org.apache.avro.reflect.Nullable; - /********************************************************************** * Protocol that a DFS datanode uses to communicate with the NameNode. * It's used to upload current load information and block reports. @@ -76,6 +74,8 @@ public interface DatanodeProtocol extend final static int DNA_RECOVERBLOCK = 6; // request a block recovery final static int DNA_ACCESSKEYUPDATE = 7; // update access key final static int DNA_BALANCERBANDWIDTHUPDATE = 8; // update balancer bandwidth + final static int DNA_UC_ACTION_REPORT_STATUS = 100; // Report upgrade status + final static int DNA_UC_ACTION_START_UPGRADE = 101; // start upgrade /** * Register Datanode. @@ -105,7 +105,6 @@ public interface DatanodeProtocol extend * @param failedVolumes number of failed volumes * @throws IOException on error */ - @Nullable public HeartbeatResponse sendHeartbeat(DatanodeRegistration registration, long capacity, long dfsUsed, long remaining, Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/UpgradeCommand.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/UpgradeCommand.java?rev=1214518&r1=1214517&r2=1214518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/UpgradeCommand.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/UpgradeCommand.java Wed Dec 14 22:47:41 2011 @@ -41,8 +41,10 @@ import org.apache.hadoop.io.WritableFact @InterfaceStability.Evolving public class UpgradeCommand extends DatanodeCommand { public final static int UC_ACTION_UNKNOWN = DatanodeProtocol.DNA_UNKNOWN; - public final static int UC_ACTION_REPORT_STATUS = 100; // report upgrade status - public final static int UC_ACTION_START_UPGRADE = 101; // start upgrade + public final static int UC_ACTION_REPORT_STATUS = + DatanodeProtocol.DNA_UC_ACTION_REPORT_STATUS; + public final static int UC_ACTION_START_UPGRADE = + DatanodeProtocol.DNA_UC_ACTION_START_UPGRADE; private int version; private short upgradeStatus; Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocolR23Compatible/DatanodeWireProtocol.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocolR23Compatible/DatanodeWireProtocol.java?rev=1214518&r1=1214517&r2=1214518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocolR23Compatible/DatanodeWireProtocol.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocolR23Compatible/DatanodeWireProtocol.java Wed Dec 14 22:47:41 2011 @@ -20,7 +20,6 @@ package org.apache.hadoop.hdfs.server.pr import java.io.IOException; -import org.apache.avro.reflect.Nullable; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.protocol.ClientProtocol; @@ -98,7 +97,6 @@ public interface DatanodeWireProtocol ex * @param failedVolumes number of failed volumes * @throws IOException on error */ - @Nullable public HeartbeatResponseWritable sendHeartbeat( DatanodeRegistrationWritable registration, long capacity, long dfsUsed, long remaining, long blockPoolUsed, int xmitsInProgress, Propchange: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/native/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Dec 14 22:47:41 2011 @@ -1,4 +1,4 @@ -/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:1152502-1213862 +/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:1152502-1214510 /hadoop/core/branches/branch-0.19/hdfs/src/main/native:713112 /hadoop/core/branches/branch-0.19/mapred/src/c++/libhdfs:713112 /hadoop/core/trunk/src/c++/libhdfs:776175-784663 Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto?rev=1214518&r1=1214517&r2=1214518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto (original) +++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto Wed Dec 14 22:47:41 2011 @@ -74,7 +74,7 @@ message AppendRequestProto { } message AppendResponseProto { - required LocatedBlockProto block = 1; + optional LocatedBlockProto block = 1; } message SetReplicationRequestProto { @@ -96,8 +96,8 @@ message SetPermissionResponseProto { // message SetOwnerRequestProto { required string src = 1; - required string username = 2; - required string groupname = 3; + optional string username = 2; + optional string groupname = 3; } message SetOwnerResponseProto { // void response @@ -139,7 +139,7 @@ message GetAdditionalDatanodeResponsePro message CompleteRequestProto { required string src = 1; required string clientName = 2; - required ExtendedBlockProto last = 3; + optional ExtendedBlockProto last = 3; } message CompleteResponseProto { @@ -204,7 +204,7 @@ message GetListingRequestProto { required bool needLocation = 3; } message GetListingResponseProto { - required DirectoryListingProto dirList = 1; + optional DirectoryListingProto dirList = 1; } message RenewLeaseRequestProto { @@ -311,7 +311,7 @@ message DistributedUpgradeProgressRespon message ListCorruptFileBlocksRequestProto { required string path = 1; - required string cookie = 2; + optional string cookie = 2; } message ListCorruptFileBlocksResponseProto { @@ -338,7 +338,7 @@ message GetFileLinkInfoRequestProto { } message GetFileLinkInfoResponseProto { - required HdfsFileStatusProto fs = 1; + optional HdfsFileStatusProto fs = 1; } message GetContentSummaryRequestProto { Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto?rev=1214518&r1=1214517&r2=1214518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto (original) +++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto Wed Dec 14 22:47:41 2011 @@ -47,6 +47,7 @@ message DatanodeCommandProto { KeyUpdateCommand = 4; RegisterCommand = 5; UpgradeCommand = 6; + NullDatanodeCommand = 7; } required Type cmdType = 1; // Type of the command @@ -80,6 +81,7 @@ message BlockCommandProto { enum Action { TRANSFER = 1; // Transfer blocks to another datanode INVALIDATE = 2; // Invalidate blocks + SHUTDOWN = 3; // Shutdown the datanode } required Action action = 1; required string blockPoolId = 2; @@ -205,7 +207,7 @@ message BlockReportRequestProto { * cmd - Command from namenode to the datanode */ message BlockReportResponseProto { - required DatanodeCommandProto cmd = 1; + optional DatanodeCommandProto cmd = 1; } /** Propchange: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Dec 14 22:47:41 2011 @@ -1,4 +1,4 @@ -/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:1159757-1213862 +/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:1159757-1214510 /hadoop/core/branches/branch-0.19/hdfs/src/main/webapps/datanode:713112 /hadoop/core/branches/branch-0.19/hdfs/src/webapps/datanode:713112 /hadoop/core/trunk/src/webapps/datanode:776175-784663 Propchange: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Dec 14 22:47:41 2011 @@ -1,4 +1,4 @@ -/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:1152502-1213862 +/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:1152502-1214510 /hadoop/core/branches/branch-0.19/hdfs/src/main/webapps/hdfs:713112 /hadoop/core/branches/branch-0.19/hdfs/src/webapps/hdfs:713112 /hadoop/core/trunk/src/webapps/hdfs:776175-784663 Propchange: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Dec 14 22:47:41 2011 @@ -1,4 +1,4 @@ -/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:1152502-1213862 +/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:1152502-1214510 /hadoop/core/branches/branch-0.19/hdfs/src/main/webapps/secondary:713112 /hadoop/core/branches/branch-0.19/hdfs/src/webapps/secondary:713112 /hadoop/core/trunk/src/webapps/secondary:776175-784663 Propchange: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Dec 14 22:47:41 2011 @@ -1,4 +1,4 @@ -/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:1159757-1213862 +/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:1159757-1214510 /hadoop/core/branches/branch-0.19/hdfs/src/test/hdfs:713112 /hadoop/core/trunk/src/test/hdfs:776175-785643 /hadoop/hdfs/branches/HDFS-1052/src/test/hdfs:987665-1095512 Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java?rev=1214518&r1=1214517&r2=1214518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java Wed Dec 14 22:47:41 2011 @@ -56,6 +56,7 @@ import org.apache.hadoop.hdfs.protocol.D import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; import org.apache.hadoop.hdfs.protocolPB.ClientDatanodeProtocolPB; +import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolPB; import org.apache.hadoop.hdfs.protocolPB.NamenodeProtocolPB; import org.apache.hadoop.hdfs.protocolR23Compatible.ClientNamenodeWireProtocol; import org.apache.hadoop.hdfs.server.common.Storage; @@ -514,37 +515,6 @@ public class MiniDFSCluster { data_dir = new File(base_dir, "data"); this.waitSafeMode = waitSafeMode; - // use alternate RPC engine if spec'd - /* - Turned off - see HDFS-2647 and HDFS-2660 for related comments. - This test can be turned on when Avro RPC is enabled using mechanism - similar to protobuf. - - String rpcEngineName = System.getProperty("hdfs.rpc.engine"); - if (rpcEngineName != null && !"".equals(rpcEngineName)) { - - LOG.info("HDFS using RPCEngine: " + rpcEngineName); - try { - Class<?> rpcEngine = conf.getClassByName(rpcEngineName); - setRpcEngine(conf, NamenodeProtocols.class, rpcEngine); - setRpcEngine(conf, ClientNamenodeWireProtocol.class, rpcEngine); - setRpcEngine(conf, ClientDatanodeProtocolPB.class, rpcEngine); - setRpcEngine(conf, NamenodeProtocolPB.class, rpcEngine); - setRpcEngine(conf, ClientProtocol.class, rpcEngine); - setRpcEngine(conf, DatanodeProtocol.class, rpcEngine); - setRpcEngine(conf, RefreshAuthorizationPolicyProtocol.class, rpcEngine); - setRpcEngine(conf, RefreshUserMappingsProtocol.class, rpcEngine); - setRpcEngine(conf, GetUserMappingsProtocol.class, rpcEngine); - } catch (ClassNotFoundException e) { - throw new RuntimeException(e); - } - - // disable service authorization, as it does not work with tunnelled RPC - conf.setBoolean(HADOOP_SECURITY_AUTHORIZATION, - false); - } - */ - int replication = conf.getInt(DFS_REPLICATION_KEY, 3); conf.setInt(DFS_REPLICATION_KEY, Math.min(replication, numDataNodes)); conf.setInt(DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, 0); Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java?rev=1214518&r1=1214517&r2=1214518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java Wed Dec 14 22:47:41 2011 @@ -31,6 +31,7 @@ import org.apache.hadoop.conf.Configurat import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB; import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodeMetrics; import org.apache.hadoop.hdfs.server.protocol.BlockCommand; import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand; @@ -67,8 +68,8 @@ public class TestBPOfferService { ((Log4JLogger)DataNode.LOG).getLogger().setLevel(Level.ALL); } - private DatanodeProtocol mockNN1; - private DatanodeProtocol mockNN2; + private DatanodeProtocolClientSideTranslatorPB mockNN1; + private DatanodeProtocolClientSideTranslatorPB mockNN2; private NNHAStatusHeartbeat[] mockHaStatuses = new NNHAStatusHeartbeat[2]; private int heartbeatCounts[] = new int[2]; private DataNode mockDn; @@ -100,8 +101,10 @@ public class TestBPOfferService { /** * Set up a mock NN with the bare minimum for a DN to register to it. */ - private DatanodeProtocol setupNNMock(int nnIdx) throws Exception { - DatanodeProtocol mock = Mockito.mock(DatanodeProtocol.class); + private DatanodeProtocolClientSideTranslatorPB setupNNMock(int nnIdx) + throws Exception { + DatanodeProtocolClientSideTranslatorPB mock = + Mockito.mock(DatanodeProtocolClientSideTranslatorPB.class); Mockito.doReturn( new NamespaceInfo(1, FAKE_CLUSTERID, FAKE_BPID, 0, HdfsConstants.LAYOUT_VERSION)) @@ -298,19 +301,21 @@ public class TestBPOfferService { * Create a BPOfferService which registers with and heartbeats with the * specified namenode proxy objects. */ - private BPOfferService setupBPOSForNNs(DatanodeProtocol ... nns) { + private BPOfferService setupBPOSForNNs( + DatanodeProtocolClientSideTranslatorPB ... nns) { // Set up some fake InetAddresses, then override the connectToNN // function to return the corresponding proxies. - final Map<InetSocketAddress, DatanodeProtocol> nnMap = Maps.newLinkedHashMap(); + final Map<InetSocketAddress, DatanodeProtocolClientSideTranslatorPB> nnMap = Maps.newLinkedHashMap(); for (int port = 0; port < nns.length; port++) { nnMap.put(new InetSocketAddress(port), nns[port]); } return new BPOfferService(Lists.newArrayList(nnMap.keySet()), mockDn) { @Override - DatanodeProtocol connectToNN(InetSocketAddress nnAddr) throws IOException { - DatanodeProtocol nn = nnMap.get(nnAddr); + DatanodeProtocolClientSideTranslatorPB connectToNN(InetSocketAddress nnAddr) + throws IOException { + DatanodeProtocolClientSideTranslatorPB nn = nnMap.get(nnAddr); if (nn == null) { throw new AssertionError("bad NN addr: " + nnAddr); } @@ -329,7 +334,7 @@ public class TestBPOfferService { }, 100, 10000); } - private void waitForBlockReport(final DatanodeProtocol mockNN) + private void waitForBlockReport(final DatanodeProtocolClientSideTranslatorPB mockNN) throws Exception { GenericTestUtils.waitFor(new Supplier<Boolean>() { @Override @@ -374,7 +379,7 @@ public class TestBPOfferService { private ReceivedDeletedBlockInfo[] waitForBlockReceived( ExtendedBlock fakeBlock, - DatanodeProtocol mockNN) throws Exception { + DatanodeProtocolClientSideTranslatorPB mockNN) throws Exception { final ArgumentCaptor<ReceivedDeletedBlockInfo[]> captor = ArgumentCaptor.forClass(ReceivedDeletedBlockInfo[].class); GenericTestUtils.waitFor(new Supplier<Boolean>() { Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDatanodeRegister.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDatanodeRegister.java?rev=1214518&r1=1214517&r2=1214518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDatanodeRegister.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDatanodeRegister.java Wed Dec 14 22:47:41 2011 @@ -23,8 +23,8 @@ import static org.junit.Assert.fail; import static org.mockito.Mockito.*; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB; import org.apache.hadoop.hdfs.server.common.IncorrectVersionException; -import org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.junit.Test; import org.mockito.Mockito; @@ -49,7 +49,8 @@ public class TestDatanodeRegister { NamespaceInfo fakeNSInfo = mock(NamespaceInfo.class); when(fakeNSInfo.getBuildVersion()).thenReturn("NSBuildVersion"); - DatanodeProtocol fakeDNProt = mock(DatanodeProtocol.class); + DatanodeProtocolClientSideTranslatorPB fakeDNProt = + mock(DatanodeProtocolClientSideTranslatorPB.class); when(fakeDNProt.versionRequest()).thenReturn(fakeNSInfo); actor.setNameNode( fakeDNProt ); Copied: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsWithMultipleNameNodes.java (from r1214510, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsWithMultipleNameNodes.java) URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsWithMultipleNameNodes.java?p2=hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsWithMultipleNameNodes.java&p1=hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsWithMultipleNameNodes.java&r1=1214510&r2=1214518&rev=1214518&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsWithMultipleNameNodes.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsWithMultipleNameNodes.java Wed Dec 14 22:47:41 2011 @@ -31,6 +31,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.MiniDFSNNTopology; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.LeaseManager; @@ -79,7 +80,7 @@ public class TestWebHdfsWithMultipleName conf.setBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY, true); cluster = new MiniDFSCluster.Builder(conf) - .numNameNodes(nNameNodes) + .nnTopology(MiniDFSNNTopology.simpleFederatedTopology(nNameNodes)) .numDataNodes(nDataNodes) .build(); cluster.waitActive();
