YARN-5162. Fix Exceptions thrown during in registerAM call when Distributed Scheduling is Enabled (Hitesh Sharma via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5b41b288 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5b41b288 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5b41b288 Branch: refs/heads/HDFS-1312 Commit: 5b41b288d01b0124664ddf6a3d6b545058bcfe6f Parents: 04ded55 Author: Arun Suresh <asur...@apache.org> Authored: Thu May 26 13:22:33 2016 -0700 Committer: Arun Suresh <asur...@apache.org> Committed: Thu May 26 14:56:37 2016 -0700 ---------------------------------------------------------------------- .../yarn/security/SchedulerSecurityInfo.java | 2 +- .../api/DistributedSchedulerProtocolPB.java | 10 +++++----- .../pb/DistSchedRegisterResponsePBImpl.java | 6 +++++- .../proto/distributed_scheduler_protocol.proto | 1 + .../TestDistributedSchedulingService.java | 20 ++++++++++++++++++++ 5 files changed, 32 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b41b288/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/SchedulerSecurityInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/SchedulerSecurityInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/SchedulerSecurityInfo.java index bfc34d3..cf25273 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/SchedulerSecurityInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/SchedulerSecurityInfo.java @@ -41,7 +41,7 @@ public class SchedulerSecurityInfo extends SecurityInfo { @Override public TokenInfo getTokenInfo(Class<?> protocol, Configuration conf) { - if (!protocol.equals(ApplicationMasterProtocolPB.class)) { + if (!ApplicationMasterProtocolPB.class.isAssignableFrom(protocol)) { return null; } return new TokenInfo() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b41b288/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/DistributedSchedulerProtocolPB.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/DistributedSchedulerProtocolPB.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/DistributedSchedulerProtocolPB.java index 413b9c9..ce7911c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/DistributedSchedulerProtocolPB.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/DistributedSchedulerProtocolPB.java @@ -21,16 +21,16 @@ package org.apache.hadoop.yarn.server.api; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.ipc.ProtocolInfo; +import org.apache.hadoop.yarn.api.ApplicationMasterProtocolPB; import org.apache.hadoop.yarn.proto.ApplicationMasterProtocol.ApplicationMasterProtocolService; - - -import org.apache.hadoop.yarn.proto.DistributedSchedulerProtocol; +import org.apache.hadoop.yarn.proto.DistributedSchedulerProtocol.DistributedSchedulerProtocolService; @Private @Unstable @ProtocolInfo(protocolName = "org.apache.hadoop.yarn.server.api.DistributedSchedulerProtocolPB", protocolVersion = 1) public interface DistributedSchedulerProtocolPB extends - DistributedSchedulerProtocol.DistributedSchedulerProtocolService.BlockingInterface, - ApplicationMasterProtocolService.BlockingInterface { + DistributedSchedulerProtocolService.BlockingInterface, + ApplicationMasterProtocolService.BlockingInterface, + ApplicationMasterProtocolPB { } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b41b288/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/DistSchedRegisterResponsePBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/DistSchedRegisterResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/DistSchedRegisterResponsePBImpl.java index 0322c70..eec62da 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/DistSchedRegisterResponsePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/DistSchedRegisterResponsePBImpl.java @@ -94,9 +94,13 @@ public class DistSchedRegisterResponsePBImpl extends DistSchedRegisterResponse { ProtoUtils.convertToProtoFormat(this.maxAllocatableCapability)); } if (this.minAllocatableCapability != null) { - builder.setMaxAllocCapability( + builder.setMinAllocCapability( ProtoUtils.convertToProtoFormat(this.minAllocatableCapability)); } + if (this.incrAllocatableCapability != null) { + builder.setIncrAllocCapability( + ProtoUtils.convertToProtoFormat(this.incrAllocatableCapability)); + } if (this.registerApplicationMasterResponse != null) { builder.setRegisterResponse( ((RegisterApplicationMasterResponsePBImpl) http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b41b288/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/distributed_scheduler_protocol.proto ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/distributed_scheduler_protocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/distributed_scheduler_protocol.proto index 7e3a77f..b94656c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/distributed_scheduler_protocol.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/distributed_scheduler_protocol.proto @@ -34,5 +34,6 @@ import "yarn_server_common_service_protos.proto"; service DistributedSchedulerProtocolService { rpc registerApplicationMasterForDistributedScheduling (RegisterApplicationMasterRequestProto) returns (DistSchedRegisterResponseProto); + rpc finishApplicationMaster (FinishApplicationMasterRequestProto) returns (FinishApplicationMasterResponseProto); rpc allocateForDistributedScheduling (AllocateRequestProto) returns (DistSchedAllocateResponseProto); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b41b288/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestDistributedSchedulingService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestDistributedSchedulingService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestDistributedSchedulingService.java index 49bf4d0..47563d5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestDistributedSchedulingService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestDistributedSchedulingService.java @@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb .RegisterApplicationMasterRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb .RegisterApplicationMasterResponsePBImpl; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.api.DistributedSchedulerProtocolPB; import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; @@ -132,6 +133,9 @@ public class TestDistributedSchedulingService { DistSchedRegisterResponse resp = factory.newRecordInstance( DistSchedRegisterResponse.class); resp.setContainerIdStart(54321l); + resp.setMaxAllocatableCapabilty(Resource.newInstance(4096, 4)); + resp.setMinAllocatableCapabilty(Resource.newInstance(1024, 1)); + resp.setIncrAllocatableCapabilty(Resource.newInstance(2048, 2)); return resp; } @@ -194,6 +198,13 @@ public class TestDistributedSchedulingService { .newRecordInstance(RegisterApplicationMasterRequest.class)) .getProto())); Assert.assertEquals(54321l, dsRegResp.getContainerIdStart()); + Assert.assertEquals(4, + dsRegResp.getMaxAllocatableCapabilty().getVirtualCores()); + Assert.assertEquals(1024, + dsRegResp.getMinAllocatableCapabilty().getMemory()); + Assert.assertEquals(2, + dsRegResp.getIncrAllocatableCapabilty().getVirtualCores()); + DistSchedAllocateResponse dsAllocResp = new DistSchedAllocateResponsePBImpl( dsProxy.allocateForDistributedScheduling(null, @@ -201,5 +212,14 @@ public class TestDistributedSchedulingService { .newRecordInstance(AllocateRequest.class)).getProto())); Assert.assertEquals( "h1", dsAllocResp.getNodesForScheduling().get(0).getHost()); + + FinishApplicationMasterResponse dsfinishResp = + new FinishApplicationMasterResponsePBImpl( + dsProxy.finishApplicationMaster(null, + ((FinishApplicationMasterRequestPBImpl) factory + .newRecordInstance(FinishApplicationMasterRequest.class)) + .getProto())); + Assert.assertEquals( + false, dsfinishResp.getIsUnregistered()); } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org