Repository: reef Updated Branches: refs/heads/master c3b350fa1 -> 8be63d230
[REEF-1838] add API to Kill Yarn Job Application from .Net client * Add API in IYarnREEFClient * Implement in YarnClient and YarnREEFClient * The code is from #1352 and implemented on top of the latest interface change JIRA: [REEF-1838](https://issues.apache.org/jira/browse/REEF-1838) Pull Request: This closes #1363 Project: http://git-wip-us.apache.org/repos/asf/reef/repo Commit: http://git-wip-us.apache.org/repos/asf/reef/commit/8be63d23 Tree: http://git-wip-us.apache.org/repos/asf/reef/tree/8be63d23 Diff: http://git-wip-us.apache.org/repos/asf/reef/diff/8be63d23 Branch: refs/heads/master Commit: 8be63d23025427a50cdbb551e16401ec308ba180 Parents: c3b350f Author: Julia Wang <[email protected]> Authored: Wed Aug 9 20:08:26 2017 -0700 Committer: Markus Weimer <[email protected]> Committed: Thu Aug 24 08:57:52 2017 -0700 ---------------------------------------------------------------------- .../Org.Apache.REEF.Client.csproj | 1 + .../YARN/IYarnREEFClient.cs | 9 ++++ .../RESTClient/DataModel/KillApplication.cs | 35 ++++++++++++++++ .../YARN/RESTClient/IYarnRMClient.cs | 15 +++++++ .../YARN/RESTClient/YarnClient.cs | 43 ++++++++++++++++++++ .../RESTClient/YarnClientNoCancellationToken.cs | 11 +++++ .../YARN/YARNREEFClient.cs | 10 +++++ .../YARN/YarnREEFDotNetClient.cs | 10 +++++ .../HelloREEFYarn.cs | 29 +++++++++++-- 9 files changed, 159 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/reef/blob/8be63d23/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj b/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj index 2c3df86..791af4f 100644 --- a/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj +++ b/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj @@ -119,6 +119,7 @@ under the License. <Compile Include="YARN\Parameters\DriverStderrFilePath.cs" /> <Compile Include="YARN\Parameters\DriverStdoutFilePath.cs" /> <Compile Include="YARN\Parameters\FileSystemUrl.cs" /> + <Compile Include="YARN\RestClient\DataModel\KillApplication.cs" /> <Compile Include="YARN\RestClient\HttpClient.cs" /> <Compile Include="YARN\RestClient\IDeserializer.cs" /> <Compile Include="YARN\RestClient\IHttpClient.cs" /> http://git-wip-us.apache.org/repos/asf/reef/blob/8be63d23/lang/cs/Org.Apache.REEF.Client/YARN/IYarnREEFClient.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/IYarnREEFClient.cs b/lang/cs/Org.Apache.REEF.Client/YARN/IYarnREEFClient.cs index e208277..37860c4 100644 --- a/lang/cs/Org.Apache.REEF.Client/YARN/IYarnREEFClient.cs +++ b/lang/cs/Org.Apache.REEF.Client/YARN/IYarnREEFClient.cs @@ -18,6 +18,7 @@ using System.Collections.Generic; using System.Threading.Tasks; using Org.Apache.REEF.Client.API; +using Org.Apache.REEF.Client.YARN.RestClient.DataModel; using Org.Apache.REEF.Utilities.Attributes; namespace Org.Apache.REEF.Client.YARN @@ -33,5 +34,13 @@ namespace Org.Apache.REEF.Client.YARN /// <returns></returns> [Unstable("0.17", "Working in progress for rest API id returned")] Task<IReadOnlyDictionary<string, IApplicationReport>> GetApplicationReports(); + + /// <summary> + /// Kills the application with specified application id. + /// </summary> + /// <param name="appId">Application id to kill.</param> + /// <returns>Returns true if the application is killed otherwise return false.</returns> + [Unstable("0.17", "Working in progress for rest API status returned")] + Task<bool> KillApplication(string appId); } } http://git-wip-us.apache.org/repos/asf/reef/blob/8be63d23/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/DataModel/KillApplication.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/DataModel/KillApplication.cs b/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/DataModel/KillApplication.cs new file mode 100644 index 0000000..96207ce --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/DataModel/KillApplication.cs @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using Newtonsoft.Json; + +namespace Org.Apache.REEF.Client.YARN.RestClient.DataModel +{ + /// <summary> + /// Class generated based on schema provided in + /// <a href="http://hadoop.apache.org/docs/r2.6.0/hadoop-yarn/hadoop-yarn-site/WebServicesIntro.html";> + /// Hadoop RM REST API</a> documentation. + /// </summary> + internal sealed class KillApplication + { + internal static readonly string Resource = @"cluster/apps/"; + internal static readonly string StateTag = @"/state"; + + [JsonProperty("state")] + public State State { get; set; } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/reef/blob/8be63d23/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/IYarnRMClient.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/IYarnRMClient.cs b/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/IYarnRMClient.cs index ee63b60..e53ceac 100644 --- a/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/IYarnRMClient.cs +++ b/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/IYarnRMClient.cs @@ -62,5 +62,20 @@ namespace Org.Apache.REEF.Client.Yarn.RestClient Task<Application> SubmitApplicationAsync( SubmitApplication submitApplicationRequest, CancellationToken cancellationToken); + + /// <summary> + /// Kills the application asynchronous. + /// </summary> + /// <param name="appId">The application identifier.</param> + /// <returns>Returns true if the application is killed otherwise return false.</returns> + Task<bool> KillApplicationAsync(string appId); + + /// <summary> + /// Kills the application asynchronous. + /// </summary> + /// <param name="appId">The application identifier.</param> + /// <param name="cancellationToken">The cancellation token.</param> + /// <returns>Returns true if the application is killed otherwise returns false.</returns> + Task<bool> KillApplicationAsync(string appId, CancellationToken cancellationToken); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/reef/blob/8be63d23/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/YarnClient.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/YarnClient.cs b/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/YarnClient.cs index 6a29c49..47d975a 100644 --- a/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/YarnClient.cs +++ b/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/YarnClient.cs @@ -17,6 +17,7 @@ using System; using System.Collections.Generic; +using System.ComponentModel; using System.Globalization; using System.Net; using System.Threading; @@ -145,6 +146,48 @@ namespace Org.Apache.REEF.Client.Yarn.RestClient return await GetApplicationAsync(submitApplication.ApplicationId, cancellationToken); } + /// <summary> + /// Kills the application asynchronous. + /// </summary> + /// <param name="appId">The application identifier.</param> + /// <param name="cancellationToken">The cancellation token.</param> + /// <returns>Returns true if the application is killed otherwise returns false.</returns> + /// <exception cref="YarnRestAPIException"></exception> + public async Task<bool> KillApplicationAsync(string appId, CancellationToken cancellationToken) + { + try + { + var killApplication = new KillApplication() + { + State = State.KILLED + }; + + var restParm = KillApplication.Resource + appId + KillApplication.StateTag; + await new RemoveSynchronizationContextAwaiter(); + var request = _requestFactory.CreateRestRequest( + restParm, + Method.PUT, + rootElement: null, + body: killApplication); + + var response = await GenerateUrlAndExecuteRequestAsync(request, cancellationToken); + Logger.Log(Level.Info, "StatusCode from response {0}", response.StatusCode); + + if (response.StatusCode != HttpStatusCode.Accepted) + { + throw new YarnRestAPIException( + string.Format("Kill Application failed with HTTP STATUS {0}", + response.StatusCode)); + } + return true; + } + catch (AggregateException e) + { + Logger.Log(Level.Error, "YarnClient:KillApplicationAsync got exception for application id {0}, {1}", appId, e); + return false; + } + } + private async Task<T> GenerateUrlAndExecuteRequestAsync<T>(RestRequest request, CancellationToken cancellationToken) { http://git-wip-us.apache.org/repos/asf/reef/blob/8be63d23/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/YarnClientNoCancellationToken.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/YarnClientNoCancellationToken.cs b/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/YarnClientNoCancellationToken.cs index 2d490f8..7492bed 100644 --- a/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/YarnClientNoCancellationToken.cs +++ b/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/YarnClientNoCancellationToken.cs @@ -78,5 +78,16 @@ namespace Org.Apache.REEF.Client.Yarn.RestClient await new RemoveSynchronizationContextAwaiter(); return await SubmitApplicationAsync(submitApplicationRequest, CancellationToken.None); } + + /// <summary> + /// Kills the application asynchronous. + /// </summary> + /// <param name="appId">The application identifier.</param> + /// <returns>Returns true if the application is killed otherwise returns false.</returns> + public async Task<bool> KillApplicationAsync(string appId) + { + await new RemoveSynchronizationContextAwaiter(); + return KillApplicationAsync(appId, CancellationToken.None).Result; + } } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/reef/blob/8be63d23/lang/cs/Org.Apache.REEF.Client/YARN/YARNREEFClient.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/YARNREEFClient.cs b/lang/cs/Org.Apache.REEF.Client/YARN/YARNREEFClient.cs index c00be21..4559c73 100644 --- a/lang/cs/Org.Apache.REEF.Client/YARN/YARNREEFClient.cs +++ b/lang/cs/Org.Apache.REEF.Client/YARN/YARNREEFClient.cs @@ -144,6 +144,16 @@ namespace Org.Apache.REEF.Client.Yarn return new ReadOnlyDictionary<string, IApplicationReport>(appReports); } + /// <summary> + /// Kills the application with specified application id. + /// </summary> + /// <param name="appId">Application id to kill.</param> + /// <returns>Returns true if the application is killed otherwise returns false.</returns> + public async Task<bool> KillApplication(string appId) + { + return await _yarnClient.KillApplicationAsync(appId); + } + private void Launch(JobRequest jobRequest, string driverFolderPath) { _driverFolderPreparationHelper.PrepareDriverFolder(jobRequest.AppParameters, driverFolderPath); http://git-wip-us.apache.org/repos/asf/reef/blob/8be63d23/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFDotNetClient.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFDotNetClient.cs b/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFDotNetClient.cs index fcae2e4..babb613 100644 --- a/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFDotNetClient.cs +++ b/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFDotNetClient.cs @@ -157,6 +157,16 @@ namespace Org.Apache.REEF.Client.YARN throw new NotImplementedException(); } + /// <summary> + /// Kills the application with specified application id. + /// </summary> + /// <param name="appId"></param> + /// <returns>Returns true if the application is killed otherwise returns false.</returns> + public async Task<bool> KillApplication(string appId) + { + return await _yarnRMClient.KillApplicationAsync(appId); + } + private SubmitApplication CreateApplicationSubmissionRequest( JobParameters jobParameters, string appId, http://git-wip-us.apache.org/repos/asf/reef/blob/8be63d23/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEFYarn.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEFYarn.cs b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEFYarn.cs index 6044569..b7564f6 100644 --- a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEFYarn.cs +++ b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEFYarn.cs @@ -95,14 +95,17 @@ namespace Org.Apache.REEF.Examples.HelloREEF .Build(); var result = _reefClient.SubmitAndGetJobStatus(helloJobRequest); - LogApplicationReport(); + + //// This is an example to Kill Job Application + //// KillApplication(result.AppId); + var state = PullFinalJobStatus(result); - Logger.Log(Level.Info, "Application state : {0}.", state); + Logger.Log(Level.Info, "Application final state : {0}.", state); } /// <summary> - /// Get application report and log. + /// Sample code to get application report and log. /// </summary> private void LogApplicationReport() { @@ -115,7 +118,25 @@ namespace Org.Apache.REEF.Examples.HelloREEF } /// <summary> - /// This is to pull job final status until the Job is done + /// Sample code to kill Job Application. + /// </summary> + /// <param name="appId">Application id to kill</param> + private void KillApplication(string appId) + { + if (_reefClient.KillApplication(appId).Result) + { + Logger.Log(Level.Info, "Application {0} is killed successfully.", appId); + } + else + { + Logger.Log(Level.Info, + "Failed to kill application {0}, possible reasons are application id is invalid or application has completed.", + appId); + } + } + + /// <summary> + /// Sample code to pull job final status until the Job is done /// </summary> /// <param name="jobSubmitionResult"></param> /// <returns></returns>
