YARN-4957. Add getNewReservation in ApplicationClientProtocol (Sean Po via curino)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/013532a9 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/013532a9 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/013532a9 Branch: refs/heads/HDFS-1312 Commit: 013532a95e63d7c53e601be530021d6d5a15ab7f Parents: 3c83cee Author: Carlo Curino <[email protected]> Authored: Wed May 25 16:55:49 2016 -0700 Committer: Carlo Curino <[email protected]> Committed: Wed May 25 16:55:49 2016 -0700 ---------------------------------------------------------------------- .../hadoop/mapred/ResourceMgrDelegate.java | 7 + .../hadoop/mapred/TestClientRedirect.java | 8 + .../yarn/api/ApplicationClientProtocol.java | 25 ++ .../GetNewReservationRequest.java | 41 ++ .../GetNewReservationResponse.java | 71 ++++ .../ReservationSubmissionRequest.java | 25 +- .../ReservationSubmissionResponse.java | 46 +-- .../main/proto/applicationclient_protocol.proto | 1 + .../src/main/proto/yarn_service_protos.proto | 9 +- .../hadoop/yarn/client/api/YarnClient.java | 23 +- .../yarn/client/api/impl/YarnClientImpl.java | 10 + .../yarn/client/api/impl/TestYarnClient.java | 370 ++++++++++++++----- .../ApplicationClientProtocolPBClientImpl.java | 19 + .../ApplicationClientProtocolPBServiceImpl.java | 21 ++ .../impl/pb/GetNewReservationRequestPBImpl.java | 77 ++++ .../pb/GetNewReservationResponsePBImpl.java | 144 ++++++++ .../pb/ReservationSubmissionRequestPBImpl.java | 30 ++ .../pb/ReservationSubmissionResponsePBImpl.java | 59 --- .../amrmproxy/MockResourceManagerFacade.java | 10 +- .../server/resourcemanager/ClientRMService.java | 36 +- .../server/resourcemanager/RMAuditLogger.java | 2 + .../reservation/ReservationInputValidator.java | 7 + .../resourcemanager/webapp/RMWebServices.java | 99 ++++- .../webapp/dao/NewReservation.java | 55 +++ .../dao/ReservationSubmissionRequestInfo.java | 11 + .../dao/ReservationSubmissionResponseInfo.java | 54 --- .../ReservationACLsTestBase.java | 61 +-- .../resourcemanager/TestClientRMService.java | 287 +++++++++----- .../TestReservationSystemWithRMHA.java | 92 +++-- .../reservation/ReservationSystemTestUtil.java | 5 +- .../webapp/TestRMWebServicesReservation.java | 351 +++++++++++++----- .../src/test/resources/submit-reservation.json | 1 + .../src/site/markdown/ReservationSystem.md | 4 +- .../src/site/markdown/ResourceManagerRest.md | 90 ++++- 34 files changed, 1634 insertions(+), 517 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/013532a9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java index 21c0d0f..af18480 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java @@ -44,6 +44,7 @@ import org.apache.hadoop.mapreduce.v2.util.MRApps; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationResponse; import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest; import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse; import org.apache.hadoop.yarn.api.protocolrecords.ReservationListRequest; @@ -436,6 +437,12 @@ public class ResourceMgrDelegate extends YarnClient { } @Override + public GetNewReservationResponse createReservation() throws YarnException, + IOException { + return client.createReservation(); + } + + @Override public ReservationSubmissionResponse submitReservation( ReservationSubmissionRequest request) throws YarnException, IOException { return client.submitReservation(request); http://git-wip-us.apache.org/repos/asf/hadoop/blob/013532a9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java index 64f967d..bed7e75 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java @@ -96,6 +96,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetLabelsToNodesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetLabelsToNodesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest; @@ -424,6 +426,12 @@ public class TestClientRedirect { } @Override + public GetNewReservationResponse getNewReservation( + GetNewReservationRequest request) throws YarnException, IOException { + return null; + } + + @Override public ReservationSubmissionResponse submitReservation( ReservationSubmissionRequest request) throws YarnException, IOException { return null; http://git-wip-us.apache.org/repos/asf/hadoop/blob/013532a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java index bca062e..82b2708 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java @@ -37,6 +37,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetLabelsToNodesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetLabelsToNodesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest; @@ -302,6 +304,28 @@ public interface ApplicationClientProtocol extends ApplicationBaseProtocol { MoveApplicationAcrossQueuesRequest request) throws YarnException, IOException; /** + * <p>The interface used by clients to obtain a new {@link ReservationId} for + * submitting new reservations.</p> + * + * <p>The <code>ResourceManager</code> responds with a new, unique, + * {@link ReservationId} which is used by the client to submit + * a new reservation.</p> + * + * @param request to get a new <code>ReservationId</code> + * @return response containing the new <code>ReservationId</code> to be used + * to submit a new reservation + * @throws YarnException if the reservation system is not enabled. + * @throws IOException on IO failures. + * @see #submitReservation(ReservationSubmissionRequest) + */ + @Public + @Unstable + @Idempotent + GetNewReservationResponse getNewReservation( + GetNewReservationRequest request) + throws YarnException, IOException; + + /** * <p> * The interface used by clients to submit a new reservation to the * {@code ResourceManager}. @@ -349,6 +373,7 @@ public interface ApplicationClientProtocol extends ApplicationBaseProtocol { */ @Public @Unstable + @Idempotent public ReservationSubmissionResponse submitReservation( ReservationSubmissionRequest request) throws YarnException, IOException; http://git-wip-us.apache.org/repos/asf/hadoop/blob/013532a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetNewReservationRequest.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetNewReservationRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetNewReservationRequest.java new file mode 100644 index 0000000..210351d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetNewReservationRequest.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.util.Records; + +/** + * <p>The request sent by clients to get a new {@code ReservationId} for + * submitting an reservation.</p> + * + * {@code ApplicationClientProtocol#getNewReservation(GetNewReservationRequest)} + */ +@Public +@Unstable +public abstract class GetNewReservationRequest { + @Public + @Unstable + public static GetNewReservationRequest newInstance() { + GetNewReservationRequest request = + Records.newRecord(GetNewReservationRequest.class); + return request; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/013532a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetNewReservationResponse.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetNewReservationResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetNewReservationResponse.java new file mode 100644 index 0000000..d7d80bf --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetNewReservationResponse.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.ReservationId; +import org.apache.hadoop.yarn.util.Records; + +/** + * <p>The response sent by the <code>ResourceManager</code> to the client for + * a request to get a new {@link ReservationId} for submitting reservations.</p> + * + * <p>Clients can submit an reservation with the returned + * {@link ReservationId}.</p> + * + * {@code ApplicationClientProtocol#getNewReservation(GetNewReservationRequest)} + */ +@Public +@Unstable +public abstract class GetNewReservationResponse { + + @Private + @Unstable + public static GetNewReservationResponse newInstance( + ReservationId reservationId) { + GetNewReservationResponse response = + Records.newRecord(GetNewReservationResponse.class); + response.setReservationId(reservationId); + return response; + } + + /** + * Get a new {@link ReservationId} to be used to submit a reservation. + * + * @return a {@link ReservationId} representing the unique id to identify + * a reservation with which it was submitted. + */ + @Public + @Unstable + public abstract ReservationId getReservationId(); + + /** + * Set a new {@link ReservationId} to be used to submit a reservation. + * + * @param reservationId a {@link ReservationId} representing the unique id to + * identify a reservation with which it was submitted. + */ + @Private + @Unstable + public abstract void setReservationId(ReservationId reservationId); + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/013532a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ReservationSubmissionRequest.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ReservationSubmissionRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ReservationSubmissionRequest.java index 6d5ca16..3872f41 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ReservationSubmissionRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ReservationSubmissionRequest.java @@ -22,6 +22,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.ReservationDefinition; +import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.util.Records; /** @@ -38,11 +39,13 @@ public abstract class ReservationSubmissionRequest { @Public @Unstable public static ReservationSubmissionRequest newInstance( - ReservationDefinition reservationDefinition, String queueName) { + ReservationDefinition reservationDefinition, String queueName, + ReservationId reservationId) { ReservationSubmissionRequest request = Records.newRecord(ReservationSubmissionRequest.class); request.setReservationDefinition(reservationDefinition); request.setQueue(queueName); + request.setReservationId(reservationId); return request; } @@ -94,4 +97,24 @@ public abstract class ReservationSubmissionRequest { @Unstable public abstract void setQueue(String queueName); + /** + * Get the reservation id that corresponds to the reservation submission. + * + * @return reservation id that will be used to identify the reservation + * submission. + */ + @Public + @Unstable + public abstract ReservationId getReservationId(); + + /** + * Set the reservation id that corresponds to the reservation submission. + * + * @param reservationId reservation id that will be used to identify the + * reservation submission. + */ + @Public + @Unstable + public abstract void setReservationId(ReservationId reservationId); + } http://git-wip-us.apache.org/repos/asf/hadoop/blob/013532a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ReservationSubmissionResponse.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ReservationSubmissionResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ReservationSubmissionResponse.java index 32fe5e0..87592d0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ReservationSubmissionResponse.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ReservationSubmissionResponse.java @@ -21,18 +21,17 @@ package org.apache.hadoop.yarn.api.protocolrecords; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.api.records.ReservationDefinition; -import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.util.Records; /** - * {@link ReservationSubmissionResponse} contains the answer of the admission - * control system in the {@code ResourceManager} to a reservation create - * operation. Response contains a {@link ReservationId} if the operation was - * successful, if not an exception reporting reason for a failure. - * - * @see ReservationDefinition - * + * <p>The response sent by the <code>ResourceManager</code> to a client on + * reservation submission.</p> + * + * <p>Currently, this is empty.</p> + * + * {@code ApplicationClientProtocol#submitReservation( + * ReservationSubmissionRequest)} + * */ @Public @Unstable @@ -40,37 +39,10 @@ public abstract class ReservationSubmissionResponse { @Private @Unstable - public static ReservationSubmissionResponse newInstance( - ReservationId reservationId) { + public static ReservationSubmissionResponse newInstance() { ReservationSubmissionResponse response = Records.newRecord(ReservationSubmissionResponse.class); - response.setReservationId(reservationId); return response; } - /** - * Get the {@link ReservationId}, that corresponds to a valid resource - * allocation in the scheduler (between start and end time of this - * reservation) - * - * @return the {@link ReservationId} representing the unique id of the - * corresponding reserved resource allocation in the scheduler - */ - @Public - @Unstable - public abstract ReservationId getReservationId(); - - /** - * Set the {@link ReservationId}, that correspond to a valid resource - * allocation in the scheduler (between start and end time of this - * reservation) - * - * @param reservationId the {@link ReservationId} representing the the unique - * id of the corresponding reserved resource allocation in the - * scheduler - */ - @Private - @Unstable - public abstract void setReservationId(ReservationId reservationId); - } http://git-wip-us.apache.org/repos/asf/hadoop/blob/013532a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto index 763c839..7046b24 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto @@ -50,6 +50,7 @@ service ApplicationClientProtocolService { rpc getApplicationAttempts (GetApplicationAttemptsRequestProto) returns (GetApplicationAttemptsResponseProto); rpc getContainerReport (GetContainerReportRequestProto) returns (GetContainerReportResponseProto); rpc getContainers (GetContainersRequestProto) returns (GetContainersResponseProto); + rpc getNewReservation (GetNewReservationRequestProto) returns (GetNewReservationResponseProto); rpc submitReservation (ReservationSubmissionRequestProto) returns (ReservationSubmissionResponseProto); rpc updateReservation (ReservationUpdateRequestProto) returns (ReservationUpdateResponseProto); rpc deleteReservation (ReservationDeleteRequestProto) returns (ReservationDeleteResponseProto); http://git-wip-us.apache.org/repos/asf/hadoop/blob/013532a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto index bdf022f..7070e38 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto @@ -367,13 +367,20 @@ message ReleaseSharedCacheResourceResponseProto { // reservation_protocol ////////////////////////////////////////////////////// +message GetNewReservationRequestProto { +} + +message GetNewReservationResponseProto { + optional ReservationIdProto reservation_id = 1; +} + message ReservationSubmissionRequestProto { optional string queue = 1; optional ReservationDefinitionProto reservation_definition = 2; + optional ReservationIdProto reservation_id = 3; } message ReservationSubmissionResponseProto { - optional ReservationIdProto reservation_id = 1; } message ReservationUpdateRequestProto { http://git-wip-us.apache.org/repos/asf/hadoop/blob/013532a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java index ff231a8..7dfe1d9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java @@ -32,6 +32,7 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.io.Text; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationResponse; import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest; import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse; import org.apache.hadoop.yarn.api.protocolrecords.ReservationListRequest; @@ -535,6 +536,20 @@ public abstract class YarnClient extends AbstractService { /** * <p> + * Obtain a {@link GetNewReservationResponse} for a new reservation, + * which contains the {@link ReservationId} object. + * </p> + * + * @return The {@link GetNewReservationResponse} containing a new + * {@link ReservationId} object. + * @throws YarnException if reservation cannot be created. + * @throws IOException if reservation cannot be created. + */ + public abstract GetNewReservationResponse createReservation() + throws YarnException, IOException; + + /** + * <p> * The interface used by clients to submit a new reservation to the * {@code ResourceManager}. * </p> @@ -666,7 +681,7 @@ public abstract class YarnClient extends AbstractService { * @return response that contains information about reservations that are * being searched for. * @throws YarnException if the request is invalid - * @throws IOException + * @throws IOException if the request failed otherwise * */ @Public @@ -725,8 +740,10 @@ public abstract class YarnClient extends AbstractService { * </p> * * @return cluster node labels collection - * @throws YarnException - * @throws IOException + * @throws YarnException when there is a failure in + * {@see ApplicationClientProtocol} + * @throws IOException when there is a failure in + * {@see ApplicationClientProtocol} */ @Public @Unstable http://git-wip-us.apache.org/repos/asf/hadoop/blob/013532a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java index 56e42c4..36e6dfe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java @@ -65,6 +65,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetLabelsToNodesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest; @@ -789,6 +791,14 @@ public class YarnClientImpl extends YarnClient { } @Override + public GetNewReservationResponse createReservation() throws YarnException, + IOException { + GetNewReservationRequest request = + Records.newRecord(GetNewReservationRequest.class); + return rmClient.getNewReservation(request); + } + + @Override public ReservationSubmissionResponse submitReservation( ReservationSubmissionRequest request) throws YarnException, IOException { return rmClient.submitReservation(request); http://git-wip-us.apache.org/repos/asf/hadoop/blob/013532a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java index 6e0930c..df1b96b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java @@ -67,6 +67,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetContainersRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetContainersResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetLabelsToNodesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetLabelsToNodesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; @@ -988,7 +990,19 @@ public class TestYarnClient { return appId; } - + + private GetNewReservationResponse getNewReservation(YarnClient rmClient) { + GetNewReservationRequest newReservationRequest = GetNewReservationRequest + .newInstance(); + GetNewReservationResponse getNewReservationResponse = null; + try { + getNewReservationResponse = rmClient.createReservation(); + } catch (Exception e) { + Assert.fail(e.getMessage()); + } + return getNewReservationResponse; + } + private void waitTillAccepted(YarnClient rmClient, ApplicationId appId, boolean unmanagedApplication) throws Exception { @@ -1179,9 +1193,7 @@ public class TestYarnClient { } } - @Test - public void testReservationAPIs() { - // initialize + private MiniYARNCluster setupMiniYARNCluster() { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); ReservationSystemTestUtil.setupQueueConfiguration(conf); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, @@ -1189,55 +1201,116 @@ public class TestYarnClient { conf.setBoolean(YarnConfiguration.RM_RESERVATION_SYSTEM_ENABLE, true); MiniYARNCluster cluster = new MiniYARNCluster("testReservationAPIs", 2, 1, 1); - YarnClient client = null; - try { - cluster.init(conf); - cluster.start(); - final Configuration yarnConf = cluster.getConfig(); - client = YarnClient.createYarnClient(); - client.init(yarnConf); - client.start(); - int attempts; - for(attempts = 10; attempts > 0; attempts--) { - if (cluster.getResourceManager().getRMContext().getReservationSystem() - .getPlan(ReservationSystemTestUtil.reservationQ).getTotalCapacity() - .getMemory() > 0) { - break; - } - try { - Thread.sleep(100); - } catch (InterruptedException e) { - e.printStackTrace(); - } + cluster.init(conf); + cluster.start(); + + int attempts; + for (attempts = 10; attempts > 0; attempts--) { + if (cluster.getResourceManager().getRMContext().getReservationSystem() + .getPlan(ReservationSystemTestUtil.reservationQ).getTotalCapacity() + .getMemory() > 6000) { + break; } - if (attempts <= 0) { - Assert.fail("Exhausted attempts in checking if node capacity was " - + "added to the plan"); + try { + Thread.sleep(100); + } catch (InterruptedException e) { + e.printStackTrace(); } + } + if (attempts <= 0) { + Assert.fail("Exhausted attempts in checking if node capacity was " + + "added to the plan"); + } - // create a reservation + return cluster; + } + + private YarnClient setupYarnClient(MiniYARNCluster cluster) { + final Configuration yarnConf = cluster.getConfig(); + YarnClient client = YarnClient.createYarnClient(); + client.init(yarnConf); + client.start(); + return client; + } + + private ReservationSubmissionRequest submitReservationTestHelper( + YarnClient client, long arrival, long deadline, long duration) { + ReservationId reservationID = getNewReservation(client).getReservationId(); + ReservationSubmissionRequest sRequest = createSimpleReservationRequest( + reservationID, 4, arrival, deadline, duration); + ReservationSubmissionResponse sResponse = null; + try { + sResponse = client.submitReservation(sRequest); + } catch (Exception e) { + Assert.fail(e.getMessage()); + } + Assert.assertNotNull(sResponse); + Assert.assertNotNull(reservationID); + System.out.println("Submit reservation response: " + reservationID); + + return sRequest; + } + + @Test + public void testCreateReservation() { + MiniYARNCluster cluster = setupMiniYARNCluster(); + YarnClient client = setupYarnClient(cluster); + try { Clock clock = new UTCClock(); long arrival = clock.getTime(); long duration = 60000; long deadline = (long) (arrival + 1.05 * duration); ReservationSubmissionRequest sRequest = - createSimpleReservationRequest(4, arrival, deadline, duration); - ReservationSubmissionResponse sResponse = null; + submitReservationTestHelper(client, arrival, deadline, duration); + + // Submit the reservation again with the same request and make sure it + // passes. try { - sResponse = client.submitReservation(sRequest); + client.submitReservation(sRequest); } catch (Exception e) { Assert.fail(e.getMessage()); } - Assert.assertNotNull(sResponse); - ReservationId reservationID = sResponse.getReservationId(); - Assert.assertNotNull(reservationID); - System.out.println("Submit reservation response: " + reservationID); - // Update the reservation + // Submit the reservation with the same reservation id but different + // reservation definition, and ensure YarnException is thrown. + arrival = clock.getTime(); + ReservationDefinition rDef = sRequest.getReservationDefinition(); + rDef.setArrival(arrival + duration); + sRequest.setReservationDefinition(rDef); + try { + client.submitReservation(sRequest); + Assert.fail("Reservation submission should fail if a duplicate " + + "reservation id is used, but the reservation definition has been " + + "updated."); + } catch (Exception e) { + Assert.assertTrue(e instanceof YarnException); + } + } finally { + // clean-up + if (client != null) { + client.stop(); + } + cluster.stop(); + } + } + + @Test + public void testUpdateReservation() { + MiniYARNCluster cluster = setupMiniYARNCluster(); + YarnClient client = setupYarnClient(cluster); + try { + Clock clock = new UTCClock(); + long arrival = clock.getTime(); + long duration = 60000; + long deadline = (long) (arrival + 1.05 * duration); + ReservationSubmissionRequest sRequest = + submitReservationTestHelper(client, arrival, deadline, duration); + ReservationDefinition rDef = sRequest.getReservationDefinition(); ReservationRequest rr = rDef.getReservationRequests().getReservationResources().get(0); + ReservationId reservationID = sRequest.getReservationId(); rr.setNumContainers(5); arrival = clock.getTime(); duration = 30000; @@ -1253,16 +1326,34 @@ public class TestYarnClient { } catch (Exception e) { Assert.fail(e.getMessage()); } - Assert.assertNotNull(sResponse); + Assert.assertNotNull(uResponse); System.out.println("Update reservation response: " + uResponse); + } finally { + // clean-up + if (client != null) { + client.stop(); + } + cluster.stop(); + } + } - // List reservations, search by reservation ID - ReservationListRequest request = - ReservationListRequest.newInstance( - ReservationSystemTestUtil.reservationQ, - reservationID.toString(), -1, -1, false); + @Test + public void testListReservationsByReservationId() { + MiniYARNCluster cluster = setupMiniYARNCluster(); + YarnClient client = setupYarnClient(cluster); + try { + Clock clock = new UTCClock(); + long arrival = clock.getTime(); + long duration = 60000; + long deadline = (long) (arrival + 1.05 * duration); + ReservationSubmissionRequest sRequest = + submitReservationTestHelper(client, arrival, deadline, duration); + ReservationId reservationID = sRequest.getReservationId(); ReservationListResponse response = null; + ReservationListRequest request = ReservationListRequest.newInstance( + ReservationSystemTestUtil.reservationQ, reservationID.toString(), -1, + -1, false); try { response = client.listReservations(request); } catch (Exception e) { @@ -1271,30 +1362,51 @@ public class TestYarnClient { Assert.assertNotNull(response); Assert.assertEquals(1, response.getReservationAllocationState().size()); Assert.assertEquals(response.getReservationAllocationState().get(0) - .getReservationId().getId(), reservationID.getId()); + .getReservationId().getId(), reservationID.getId()); Assert.assertEquals(response.getReservationAllocationState().get(0) - .getResourceAllocationRequests().size(), 0); + .getResourceAllocationRequests().size(), 0); + } finally { + // clean-up + if (client != null) { + client.stop(); + } + cluster.stop(); + } + } - // List reservations, search by time interval. - request = ReservationListRequest.newInstance( - ReservationSystemTestUtil.reservationQ, "", arrival + - duration/2, arrival + duration/2, true); + @Test + public void testListReservationsByTimeInterval() { + MiniYARNCluster cluster = setupMiniYARNCluster(); + YarnClient client = setupYarnClient(cluster); + try { + Clock clock = new UTCClock(); + long arrival = clock.getTime(); + long duration = 60000; + long deadline = (long) (arrival + 1.05 * duration); + ReservationSubmissionRequest sRequest = + submitReservationTestHelper(client, arrival, deadline, duration); - response = null; + // List reservations, search by a point in time within the reservation + // range. + arrival = clock.getTime(); + ReservationId reservationID = sRequest.getReservationId(); + ReservationListRequest request = ReservationListRequest.newInstance( + ReservationSystemTestUtil.reservationQ, "", arrival + duration / 2, + arrival + duration / 2, true); + + ReservationListResponse response = null; try { - response = client.listReservations(request); + response = client.listReservations(request); } catch (Exception e) { Assert.fail(e.getMessage()); } Assert.assertNotNull(response); Assert.assertEquals(1, response.getReservationAllocationState().size()); Assert.assertEquals(response.getReservationAllocationState().get(0) - .getReservationId().getId(), reservationID.getId()); - - // List reservations, search by invalid end time == -1. + .getReservationId().getId(), reservationID.getId()); + // List reservations, search by time within reservation interval. request = ReservationListRequest.newInstance( - ReservationSystemTestUtil.reservationQ, "", 1, -1, - true); + ReservationSystemTestUtil.reservationQ, "", 1, Long.MAX_VALUE, true); response = null; try { @@ -1305,14 +1417,45 @@ public class TestYarnClient { Assert.assertNotNull(response); Assert.assertEquals(1, response.getReservationAllocationState().size()); Assert.assertEquals(response.getReservationAllocationState().get(0) - .getReservationId().getId(), reservationID.getId()); + .getReservationId().getId(), reservationID.getId()); + // Verify that the full resource allocations exist. + Assert.assertTrue(response.getReservationAllocationState().get(0) + .getResourceAllocationRequests().size() > 0); - // List reservations, search by invalid end time < -1. - request = ReservationListRequest.newInstance( - ReservationSystemTestUtil.reservationQ, "", 1, -10, - true); + // Verify that the full RDL is returned. + ReservationRequests reservationRequests = + response.getReservationAllocationState().get(0) + .getReservationDefinition().getReservationRequests(); + Assert.assertTrue( + reservationRequests.getInterpreter().toString().equals("R_ALL")); + Assert.assertTrue(reservationRequests.getReservationResources().get(0) + .getDuration() == duration); + } finally { + // clean-up + if (client != null) { + client.stop(); + } + cluster.stop(); + } + } - response = null; + @Test + public void testListReservationsByInvalidTimeInterval() { + MiniYARNCluster cluster = setupMiniYARNCluster(); + YarnClient client = setupYarnClient(cluster); + try { + Clock clock = new UTCClock(); + long arrival = clock.getTime(); + long duration = 60000; + long deadline = (long) (arrival + 1.05 * duration); + ReservationSubmissionRequest sRequest = + submitReservationTestHelper(client, arrival, deadline, duration); + + // List reservations, search by invalid end time == -1. + ReservationListRequest request = ReservationListRequest + .newInstance(ReservationSystemTestUtil.reservationQ, "", 1, -1, true); + + ReservationListResponse response = null; try { response = client.listReservations(request); } catch (Exception e) { @@ -1321,12 +1464,11 @@ public class TestYarnClient { Assert.assertNotNull(response); Assert.assertEquals(1, response.getReservationAllocationState().size()); Assert.assertEquals(response.getReservationAllocationState().get(0) - .getReservationId().getId(), reservationID.getId()); + .getReservationId().getId(), sRequest.getReservationId().getId()); - // List reservations, search by time within reservation interval. + // List reservations, search by invalid end time < -1. request = ReservationListRequest.newInstance( - ReservationSystemTestUtil.reservationQ, "", 1, Long.MAX_VALUE, - true); + ReservationSystemTestUtil.reservationQ, "", 1, -10, true); response = null; try { @@ -1337,38 +1479,52 @@ public class TestYarnClient { Assert.assertNotNull(response); Assert.assertEquals(1, response.getReservationAllocationState().size()); Assert.assertEquals(response.getReservationAllocationState().get(0) - .getReservationId().getId(), reservationID.getId()); - - // Verify that the full resource allocations exist. - Assert.assertTrue(response.getReservationAllocationState().get(0) - .getResourceAllocationRequests().size() > 0); + .getReservationId().getId(), sRequest.getReservationId().getId()); + } finally { + // clean-up + if (client != null) { + client.stop(); + } + cluster.stop(); + } + } - // Verify that the full RDL is returned. - ReservationRequests reservationRequests = response - .getReservationAllocationState().get(0) - .getReservationDefinition().getReservationRequests(); - Assert.assertTrue(reservationRequests.getInterpreter().toString() - .equals("R_ALL")); - Assert.assertTrue(reservationRequests.getReservationResources().get(0) - .getDuration() == duration); + @Test + public void testListReservationsByTimeIntervalContainingNoReservations() { + MiniYARNCluster cluster = setupMiniYARNCluster(); + YarnClient client = setupYarnClient(cluster); + try { + Clock clock = new UTCClock(); + long arrival = clock.getTime(); + long duration = 60000; + long deadline = (long) (arrival + 1.05 * duration); + ReservationSubmissionRequest sRequest = + submitReservationTestHelper(client, arrival, deadline, duration); // List reservations, search by very large start time. - request = ReservationListRequest.newInstance( - ReservationSystemTestUtil.reservationQ, "", Long.MAX_VALUE, - -1, false); + ReservationListRequest request = ReservationListRequest.newInstance( + ReservationSystemTestUtil.reservationQ, "", Long.MAX_VALUE, -1, + false); - response = null; + ReservationListResponse response = null; try { response = client.listReservations(request); } catch (Exception e) { Assert.fail(e.getMessage()); } + // Ensure all reservations are filtered out. + Assert.assertNotNull(response); + Assert.assertEquals(response.getReservationAllocationState().size(), 0); + + duration = 30000; + deadline = sRequest.getReservationDefinition().getDeadline(); + // List reservations, search by start time after the reservation // end time. request = ReservationListRequest.newInstance( - ReservationSystemTestUtil.reservationQ, "", deadline + duration, - deadline + 2 * duration, false); + ReservationSystemTestUtil.reservationQ, "", deadline + duration, + deadline + 2 * duration, false); response = null; try { @@ -1381,11 +1537,12 @@ public class TestYarnClient { Assert.assertNotNull(response); Assert.assertEquals(response.getReservationAllocationState().size(), 0); + arrival = clock.getTime(); // List reservations, search by end time before the reservation start // time. request = ReservationListRequest.newInstance( - ReservationSystemTestUtil.reservationQ, "", 0, arrival - - duration, false); + ReservationSystemTestUtil.reservationQ, "", 0, arrival - duration, + false); response = null; try { @@ -1399,8 +1556,8 @@ public class TestYarnClient { Assert.assertEquals(response.getReservationAllocationState().size(), 0); // List reservations, search by very small end time. - request = ReservationListRequest.newInstance( - ReservationSystemTestUtil.reservationQ, "", 0, 1, false); + request = ReservationListRequest + .newInstance(ReservationSystemTestUtil.reservationQ, "", 0, 1, false); response = null; try { @@ -1413,6 +1570,28 @@ public class TestYarnClient { Assert.assertNotNull(response); Assert.assertEquals(response.getReservationAllocationState().size(), 0); + } finally { + // clean-up + if (client != null) { + client.stop(); + } + cluster.stop(); + } + } + + @Test + public void testReservationDelete() { + MiniYARNCluster cluster = setupMiniYARNCluster(); + YarnClient client = setupYarnClient(cluster); + try { + Clock clock = new UTCClock(); + long arrival = clock.getTime(); + long duration = 60000; + long deadline = (long) (arrival + 1.05 * duration); + ReservationSubmissionRequest sRequest = + submitReservationTestHelper(client, arrival, deadline, duration); + + ReservationId reservationID = sRequest.getReservationId(); // Delete the reservation ReservationDeleteRequest dRequest = ReservationDeleteRequest.newInstance(reservationID); @@ -1422,15 +1601,15 @@ public class TestYarnClient { } catch (Exception e) { Assert.fail(e.getMessage()); } - Assert.assertNotNull(sResponse); + Assert.assertNotNull(dResponse); System.out.println("Delete reservation response: " + dResponse); // List reservations, search by non-existent reservationID - request = ReservationListRequest.newInstance( - ReservationSystemTestUtil.reservationQ, - reservationID.toString(), -1, -1, false); + ReservationListRequest request = ReservationListRequest.newInstance( + ReservationSystemTestUtil.reservationQ, reservationID.toString(), -1, + -1, false); - response = null; + ReservationListResponse response = null; try { response = client.listReservations(request); } catch (Exception e) { @@ -1448,7 +1627,8 @@ public class TestYarnClient { } private ReservationSubmissionRequest createSimpleReservationRequest( - int numContainers, long arrival, long deadline, long duration) { + ReservationId reservationId, int numContainers, long arrival, + long deadline, long duration) { // create a request with a single atomic ask ReservationRequest r = ReservationRequest.newInstance(Resource.newInstance(1024, 1), @@ -1461,7 +1641,7 @@ public class TestYarnClient { "testYarnClient#reservation"); ReservationSubmissionRequest request = ReservationSubmissionRequest.newInstance(rDef, - ReservationSystemTestUtil.reservationQ); + ReservationSystemTestUtil.reservationQ, reservationId); return request; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/013532a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java index e5aad74..4d65425 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java @@ -59,6 +59,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetLabelsToNodesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetLabelsToNodesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest; @@ -113,6 +115,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetLabelsToNodesReques import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetLabelsToNodesResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetNewApplicationRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetNewApplicationResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetNewReservationRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetNewReservationResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetNodesToLabelsRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetNodesToLabelsResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetQueueInfoRequestPBImpl; @@ -453,6 +457,21 @@ public class ApplicationClientProtocolPBClientImpl implements ApplicationClientP } @Override + public GetNewReservationResponse getNewReservation( + GetNewReservationRequest request) + throws YarnException, IOException { + YarnServiceProtos.GetNewReservationRequestProto requestProto = + ((GetNewReservationRequestPBImpl) request).getProto(); + try { + return new GetNewReservationResponsePBImpl(proxy.getNewReservation(null, + requestProto)); + } catch (ServiceException e) { + RPCUtil.unwrapAndThrowException(e); + return null; + } + } + + @Override public ReservationSubmissionResponse submitReservation(ReservationSubmissionRequest request) throws YarnException, IOException { ReservationSubmissionRequestProto requestProto = http://git-wip-us.apache.org/repos/asf/hadoop/blob/013532a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java index 2c5794e..4923794 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java @@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetContainersResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetLabelsToNodesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse; @@ -84,6 +85,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetLabelsToNodesReques import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetLabelsToNodesResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetNewApplicationRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetNewApplicationResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetNewReservationRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetNewReservationResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetNodesToLabelsRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetNodesToLabelsResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetQueueInfoRequestPBImpl; @@ -136,6 +139,8 @@ import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetLabelsToNodesRequestPro import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetLabelsToNodesResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetNewApplicationRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetNewApplicationResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetNewReservationRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetNewReservationResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetNodesToLabelsRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetNodesToLabelsResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetQueueInfoRequestProto; @@ -449,6 +454,22 @@ public class ApplicationClientProtocolPBServiceImpl implements ApplicationClient } @Override + public GetNewReservationResponseProto getNewReservation( + RpcController arg0, GetNewReservationRequestProto proto) throws + ServiceException { + GetNewReservationRequestPBImpl request = + new GetNewReservationRequestPBImpl(proto); + try { + GetNewReservationResponse response = real.getNewReservation(request); + return ((GetNewReservationResponsePBImpl)response).getProto(); + } catch (YarnException e) { + throw new ServiceException(e); + } catch (IOException e) { + throw new ServiceException(e); + } + } + + @Override public ReservationSubmissionResponseProto submitReservation(RpcController controller, ReservationSubmissionRequestProto requestProto) throws ServiceException { ReservationSubmissionRequestPBImpl request = http://git-wip-us.apache.org/repos/asf/hadoop/blob/013532a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetNewReservationRequestPBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetNewReservationRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetNewReservationRequestPBImpl.java new file mode 100644 index 0000000..b43f6b5 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetNewReservationRequestPBImpl.java @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + + +import com.google.protobuf.TextFormat; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationRequest; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetNewReservationRequestProto; + +/** + * <p>The implementation of the request sent by clients to get a + * new {@code ReservationId} for submitting an reservation.</p> + * + * {@code ApplicationClientProtocol#getNewReservation(GetNewReservationRequest)} + */ +@Private +@Unstable +public class GetNewReservationRequestPBImpl extends GetNewReservationRequest { + private GetNewReservationRequestProto proto = + GetNewReservationRequestProto.getDefaultInstance(); + private GetNewReservationRequestProto.Builder builder = null; + private boolean viaProto = false; + + public GetNewReservationRequestPBImpl(GetNewReservationRequestProto proto) { + this.proto = proto; + viaProto = true; + } + + public GetNewReservationRequestPBImpl() { + builder = GetNewReservationRequestProto.newBuilder(); + } + + public GetNewReservationRequestProto getProto() { + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/013532a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetNewReservationResponsePBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetNewReservationResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetNewReservationResponsePBImpl.java new file mode 100644 index 0000000..c09c4e2 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetNewReservationResponsePBImpl.java @@ -0,0 +1,144 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + + +import com.google.protobuf.TextFormat; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationResponse; +import org.apache.hadoop.yarn.api.records.ReservationId; +import org.apache.hadoop.yarn.api.records.impl.pb.ReservationIdPBImpl; +import org.apache.hadoop.yarn.proto.YarnProtos.ReservationIdProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetNewReservationResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetNewReservationResponseProtoOrBuilder; + + +/** + * <p>The implementation of the response sent by the + * <code>ResourceManager</code> to the client for a request to get a new + * {@link ReservationId} for submitting reservations.</p> + * + * <p>Clients can submit an reservation with the returned + * {@link ReservationId}.</p> + * + * {@code ApplicationClientProtocol#getNewReservation(GetNewReservationRequest)} + */ +@Private +@Unstable +public class GetNewReservationResponsePBImpl extends GetNewReservationResponse { + private GetNewReservationResponseProto proto = + GetNewReservationResponseProto.getDefaultInstance(); + private GetNewReservationResponseProto.Builder builder = null; + private boolean viaProto = false; + + private ReservationId reservationId = null; + + public GetNewReservationResponsePBImpl() { + builder = GetNewReservationResponseProto.newBuilder(); + } + + public GetNewReservationResponsePBImpl(GetNewReservationResponseProto proto) { + this.proto = proto; + viaProto = true; + } + + public GetNewReservationResponseProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } + + private void mergeLocalToBuilder() { + if (reservationId != null) { + builder.setReservationId(convertToProtoFormat(this.reservationId)); + } + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = GetNewReservationResponseProto.newBuilder(proto); + } + viaProto = false; + } + + @Override + public ReservationId getReservationId() { + if (this.reservationId != null) { + return this.reservationId; + } + + GetNewReservationResponseProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasReservationId()) { + return null; + } + + this.reservationId = convertFromProtoFormat(p.getReservationId()); + return this.reservationId; + } + + @Override + public void setReservationId(ReservationId reservationId) { + maybeInitBuilder(); + if (reservationId == null) { + builder.clearReservationId(); + } + this.reservationId = reservationId; + } + + private ReservationIdPBImpl convertFromProtoFormat(ReservationIdProto p) { + return new ReservationIdPBImpl(p); + } + + private ReservationIdProto convertToProtoFormat(ReservationId t) { + return ((ReservationIdPBImpl)t).getProto(); + } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/013532a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ReservationSubmissionRequestPBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ReservationSubmissionRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ReservationSubmissionRequestPBImpl.java index 9f10f4c..016d4de 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ReservationSubmissionRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ReservationSubmissionRequestPBImpl.java @@ -20,8 +20,11 @@ package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest; import org.apache.hadoop.yarn.api.records.ReservationDefinition; +import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.api.records.impl.pb.ReservationDefinitionPBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.ReservationIdPBImpl; import org.apache.hadoop.yarn.proto.YarnProtos.ReservationDefinitionProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ReservationIdProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReservationSubmissionRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReservationSubmissionRequestProtoOrBuilder; @@ -119,6 +122,25 @@ public class ReservationSubmissionRequestPBImpl extends builder.setQueue(planName); } + @Override + public ReservationId getReservationId() { + ReservationSubmissionRequestProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasReservationId()) { + return null; + } + return (convertFromProtoFormat(p.getReservationId())); + } + + @Override + public void setReservationId(ReservationId reservationId) { + maybeInitBuilder(); + if (reservationId == null) { + builder.clearReservationId(); + return; + } + builder.setReservationId(convertToProtoFormat(reservationId)); + } + private ReservationDefinitionProto convertToProtoFormat( ReservationDefinition r) { return ((ReservationDefinitionPBImpl) r).getProto(); @@ -129,6 +151,14 @@ public class ReservationSubmissionRequestPBImpl extends return new ReservationDefinitionPBImpl(r); } + private ReservationIdProto convertToProtoFormat(ReservationId r) { + return ((ReservationIdPBImpl) r).getProto(); + } + + private ReservationIdPBImpl convertFromProtoFormat(ReservationIdProto r) { + return new ReservationIdPBImpl(r); + } + @Override public int hashCode() { return getProto().hashCode(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/013532a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ReservationSubmissionResponsePBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ReservationSubmissionResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ReservationSubmissionResponsePBImpl.java index 82151f4..81b3a84 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ReservationSubmissionResponsePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ReservationSubmissionResponsePBImpl.java @@ -19,11 +19,7 @@ package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionResponse; -import org.apache.hadoop.yarn.api.records.ReservationId; -import org.apache.hadoop.yarn.api.records.impl.pb.ReservationIdPBImpl; -import org.apache.hadoop.yarn.proto.YarnProtos.ReservationIdProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReservationSubmissionResponseProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReservationSubmissionResponseProtoOrBuilder; import com.google.protobuf.TextFormat; @@ -35,8 +31,6 @@ public class ReservationSubmissionResponsePBImpl extends ReservationSubmissionResponseProto.Builder builder = null; boolean viaProto = false; - private ReservationId reservationId; - public ReservationSubmissionResponsePBImpl() { builder = ReservationSubmissionResponseProto.newBuilder(); } @@ -48,64 +42,11 @@ public class ReservationSubmissionResponsePBImpl extends } public ReservationSubmissionResponseProto getProto() { - mergeLocalToProto(); proto = viaProto ? proto : builder.build(); viaProto = true; return proto; } - private void mergeLocalToBuilder() { - if (this.reservationId != null) { - builder.setReservationId(convertToProtoFormat(this.reservationId)); - } - } - - private void mergeLocalToProto() { - if (viaProto) - maybeInitBuilder(); - mergeLocalToBuilder(); - proto = builder.build(); - viaProto = true; - } - - private void maybeInitBuilder() { - if (viaProto || builder == null) { - builder = ReservationSubmissionResponseProto.newBuilder(proto); - } - viaProto = false; - } - - @Override - public ReservationId getReservationId() { - ReservationSubmissionResponseProtoOrBuilder p = viaProto ? proto : builder; - if (reservationId != null) { - return reservationId; - } - if (!p.hasReservationId()) { - return null; - } - reservationId = convertFromProtoFormat(p.getReservationId()); - return reservationId; - } - - @Override - public void setReservationId(ReservationId reservationId) { - maybeInitBuilder(); - if (reservationId == null) { - builder.clearReservationId(); - return; - } - this.reservationId = reservationId; - } - - private ReservationIdPBImpl convertFromProtoFormat(ReservationIdProto p) { - return new ReservationIdPBImpl(p); - } - - private ReservationIdProto convertToProtoFormat(ReservationId t) { - return ((ReservationIdPBImpl) t).getProto(); - } - @Override public int hashCode() { return getProto().hashCode(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/013532a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/MockResourceManagerFacade.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/MockResourceManagerFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/MockResourceManagerFacade.java index c512f8d..6420cb0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/MockResourceManagerFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/MockResourceManagerFacade.java @@ -61,6 +61,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetLabelsToNodesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetLabelsToNodesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest; @@ -426,6 +428,12 @@ public class MockResourceManagerFacade implements } @Override + public GetNewReservationResponse getNewReservation( + GetNewReservationRequest request) throws YarnException, IOException { + throw new NotImplementedException(); + } + + @Override public ReservationSubmissionResponse submitReservation( ReservationSubmissionRequest request) throws YarnException, IOException { @@ -488,4 +496,4 @@ return null; FailApplicationAttemptRequest request) throws YarnException, IOException { throw new NotImplementedException(); } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/013532a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index d68b7b1..aeeea2b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -79,6 +79,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetLabelsToNodesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetLabelsToNodesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest; @@ -1218,18 +1220,47 @@ public class ClientRMService extends AbstractService implements } @Override + public GetNewReservationResponse getNewReservation( + GetNewReservationRequest request) throws YarnException, IOException { + checkReservationSytem(AuditConstants.CREATE_NEW_RESERVATION_REQUEST); + GetNewReservationResponse response = + recordFactory.newRecordInstance(GetNewReservationResponse.class); + + ReservationId reservationId = reservationSystem.getNewReservationId(); + response.setReservationId(reservationId); + // Create a new Reservation Id + return response; + } + + @Override public ReservationSubmissionResponse submitReservation( ReservationSubmissionRequest request) throws YarnException, IOException { // Check if reservation system is enabled checkReservationSytem(AuditConstants.SUBMIT_RESERVATION_REQUEST); ReservationSubmissionResponse response = recordFactory.newRecordInstance(ReservationSubmissionResponse.class); - // Create a new Reservation Id - ReservationId reservationId = reservationSystem.getNewReservationId(); + ReservationId reservationId = request.getReservationId(); // Validate the input Plan plan = rValidator.validateReservationSubmissionRequest(reservationSystem, request, reservationId); + + ReservationAllocation allocation = plan.getReservationById(reservationId); + + if (allocation != null) { + boolean isNewDefinition = !allocation.getReservationDefinition().equals( + request.getReservationDefinition()); + if (isNewDefinition) { + String message = "Reservation allocation already exists with the " + + "reservation id " + reservationId.toString() + ", but a different" + + " reservation definition was provided. Please try again with a " + + "new reservation id, or consider updating the reservation instead."; + throw RPCUtil.getRemoteException(message); + } else { + return response; + } + } + // Check ACLs String queueName = request.getQueue(); String user = @@ -1248,7 +1279,6 @@ public class ClientRMService extends AbstractService implements refreshScheduler(queueName, request.getReservationDefinition(), reservationId.toString()); // return the reservation id - response.setReservationId(reservationId); } } catch (PlanningException e) { RMAuditLogger.logFailure(user, AuditConstants.SUBMIT_RESERVATION_REQUEST, http://git-wip-us.apache.org/repos/asf/hadoop/blob/013532a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java index d08cb9e..c5bf000 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java @@ -69,6 +69,8 @@ public class RMAuditLogger { public static final String UNAUTHORIZED_USER = "Unauthorized user"; // For Reservation system + public static final String CREATE_NEW_RESERVATION_REQUEST = "Create " + + "Reservation Request"; public static final String SUBMIT_RESERVATION_REQUEST = "Submit Reservation Request"; public static final String UPDATE_RESERVATION_REQUEST = "Update Reservation Request"; public static final String DELETE_RESERVATION_REQUEST = "Delete Reservation Request"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/013532a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationInputValidator.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationInputValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationInputValidator.java index d63e725..8ebde0a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationInputValidator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationInputValidator.java @@ -206,10 +206,17 @@ public class ReservationInputValidator { ReservationSystem reservationSystem, ReservationSubmissionRequest request, ReservationId reservationId) throws YarnException { + String message; + if (reservationId == null) { + message = "Reservation id cannot be null. Please try again " + + "specifying a valid reservation id by creating a new reservation id."; + throw RPCUtil.getRemoteException(message); + } // Check if it is a managed queue String queue = request.getQueue(); Plan plan = getPlanFromQueue(reservationSystem, queue, AuditConstants.SUBMIT_RESERVATION_REQUEST); + validateReservationDefinition(reservationId, request.getReservationDefinition(), plan, AuditConstants.SUBMIT_RESERVATION_REQUEST); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
