Repository: reef Updated Branches: refs/heads/master 721f8ca48 -> 47593a770
[REEF-1823] Support YARN API GetApplications() in .Net YarnREEFClient This addressed the issue by * Exposing GetApplications in IYarnRMClient * Adding a IYarnREEFClient for Yarn specific functions * Implementing the new functionality in IYarnREEFClient * Updating YARNClientConfiguration to bind IYarnREEFClient to YarnREEFClient * Continuing to bind IREEFClient also to YarnREEFClient for backward compatibility * Adding test cases * Updating HelloREEFYarn to use the new API JIRA: [REEF-1823](https://issues.apache.org/jira/browse/REEF-1823) Pull request: This closes #1361 Project: http://git-wip-us.apache.org/repos/asf/reef/repo Commit: http://git-wip-us.apache.org/repos/asf/reef/commit/47593a77 Tree: http://git-wip-us.apache.org/repos/asf/reef/tree/47593a77 Diff: http://git-wip-us.apache.org/repos/asf/reef/diff/47593a77 Branch: refs/heads/master Commit: 47593a77059e082b1e094af3b6811473cb5ac628 Parents: 721f8ca Author: Julia Wang <[email protected]> Authored: Tue Aug 8 23:17:59 2017 -0700 Committer: Shravan Narayanamurthy <[email protected]> Committed: Wed Aug 9 18:23:03 2017 -0700 ---------------------------------------------------------------------- .../YarnClientTests.cs | 63 ++++++++++++ .../Org.Apache.REEF.Client.csproj | 3 + .../YARN/ApplicationReport.cs | 101 +++++++++++++++++++ .../YARN/IApplicationReport.cs | 71 +++++++++++++ .../YARN/IYarnREEFClient.cs | 37 +++++++ .../YARN/RESTClient/DataModel/Application.cs | 33 ++++++ .../YARN/RESTClient/IYarnRMClient.cs | 16 +++ .../YARN/RESTClient/YarnClient.cs | 19 ++++ .../RESTClient/YarnClientNoCancellationToken.cs | 25 +++++ .../YARN/YARNClientConfiguration.cs | 1 + .../YARN/YARNREEFClient.cs | 47 ++++++++- .../YARN/YarnREEFDotNetClient.cs | 13 ++- .../HelloREEFYarn.cs | 20 +++- 13 files changed, 442 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/reef/blob/47593a77/lang/cs/Org.Apache.REEF.Client.Tests/YarnClientTests.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client.Tests/YarnClientTests.cs b/lang/cs/Org.Apache.REEF.Client.Tests/YarnClientTests.cs index 3858b12..127ce52 100644 --- a/lang/cs/Org.Apache.REEF.Client.Tests/YarnClientTests.cs +++ b/lang/cs/Org.Apache.REEF.Client.Tests/YarnClientTests.cs @@ -132,6 +132,69 @@ namespace Org.Apache.REEF.Client.Tests } [Fact] + public async Task TestGetApplications() + { + var ctx = new TestContext(); + var urlProvider = ctx.UrlProviderFake; + var restReqExecutor = ctx.RestRequestExecutorFake; + var anyUri = Enumerable.Repeat(new Uri("anyscheme://anypath"), 1); + + urlProvider.GetUrlAsync().Returns(Task.FromResult(anyUri)); + var anyApplications = new Applications(); + + var anyApplication1 = new Application + { + AllocatedMB = 100, + AmHostHttpAddress = "http://anyhttpaddress";, + AmContainerLogs = "SomeLogs", + ApplicationType = "AnyYarnApplicationType", + State = State.FINISHED, + Name = "AnyApplicationName", + RunningContainers = 0 + }; + + var anyApplication2 = new Application + { + AllocatedMB = 100, + AmHostHttpAddress = "http://anyhttpaddress";, + AmContainerLogs = "SomeLogs", + ApplicationType = "AnyYarnApplicationType", + State = State.FINISHED, + Name = "AnyApplicationName", + RunningContainers = 0 + }; + + anyApplications.App = new List<Application> { anyApplication1, anyApplication2 }; + + restReqExecutor.ExecuteAsync<Applications>( + Arg.Is<RestRequest>( + req => + req.Resource == "ws/v1/cluster/apps" + && req.RootElement == Applications.RootElement + && req.Method == Method.GET), + anyUri.First(), + CancellationToken.None).Returns(Task.FromResult(anyApplications)); + + var yarnClient = ctx.GetClient(); + Applications actualApplications = await yarnClient.GetApplicationsAsync(); + + Assert.NotEqual(actualApplications.App, null); + Assert.Equal(actualApplications.App.Count, 2); + + int matchCount = 0; + foreach (var anyApplication in actualApplications.App) + { + if (anyApplication == anyApplication1 || anyApplication == anyApplication2) + { + ++matchCount; + } + } + + Assert.Equal(actualApplications.App.Count, matchCount); + var unused = urlProvider.Received(1).GetUrlAsync(); + } + + [Fact] public async Task TestGetApplicationFinalStatus() { var ctx = new TestContext(); http://git-wip-us.apache.org/repos/asf/reef/blob/47593a77/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj b/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj index c8df560..2c3df86 100644 --- a/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj +++ b/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj @@ -99,18 +99,21 @@ under the License. <Compile Include="Local\Parameters\LocalRuntimeDirectory.cs" /> <Compile Include="Local\Parameters\NumberOfEvaluators.cs" /> <Compile Include="Properties\AssemblyInfo.cs" /> + <Compile Include="YARN\ApplicationReport.cs" /> <Compile Include="YARN\HDI\HDInsightClientConfiguration.cs" /> <Compile Include="YARN\HDI\HDInsightCommandLineEnvironment.cs" /> <Compile Include="YARN\HDI\HDInsightCredential.cs" /> <Compile Include="YARN\HDI\HDInsightResourceFileRemoteUrlToClusterUrlConverter.cs" /> <Compile Include="YARN\HDI\HDInsightUrlProvider.cs" /> <Compile Include="YARN\HDI\NamedParameters.cs" /> + <Compile Include="YARN\IApplicationReport.cs" /> <Compile Include="YARN\IdentityResourceFileRemoteUrlToClusterUrlConverter.cs" /> <Compile Include="YARN\IResourceFileRemoteUrlToClusterUrlConverter.cs" /> <Compile Include="YARN\IYarnCommandLineEnvironment.cs" /> <Compile Include="YARN\IYarnJobCommandProvider.cs" /> <Compile Include="YARN\IJobResourceUploader.cs" /> <Compile Include="YARN\IJobSubmissionDirectoryProvider.cs" /> + <Compile Include="YARN\IYarnREEFClient.cs" /> <Compile Include="YARN\Parameters\DriverMaxMemoryAllicationPoolSizeMB.cs" /> <Compile Include="YARN\Parameters\DriverMaxPermSizeMB.cs" /> <Compile Include="YARN\Parameters\DriverStderrFilePath.cs" /> http://git-wip-us.apache.org/repos/asf/reef/blob/47593a77/lang/cs/Org.Apache.REEF.Client/YARN/ApplicationReport.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/ApplicationReport.cs b/lang/cs/Org.Apache.REEF.Client/YARN/ApplicationReport.cs new file mode 100644 index 0000000..6cbd89c --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Client/YARN/ApplicationReport.cs @@ -0,0 +1,101 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System; +using Org.Apache.REEF.Client.YARN.RestClient.DataModel; +using Org.Apache.REEF.Utilities.Logging; + +namespace Org.Apache.REEF.Client.YARN +{ + /// <summary> + /// This class represents application information maintained by YARN RM. + /// This class is modeled on Org.Apache.REEF.Client.YARN.RestClient.DataModel.Application. + /// Documentation on the class Org.Apache.REEF.Client.YARN.RestClient.DataModel.Application + /// can be found here. + /// <a href="http://hadoop.apache.org/docs/r2.6.0/hadoop-yarn/hadoop-yarn-site/WebServicesIntro.html";> + /// Hadoop RM REST API</a> documentation. + /// </summary> + internal sealed class ApplicationReport : IApplicationReport + { + /// <summary> + /// Application name. + /// </summary> + public string AppName { get; private set; } + + /// <summary> + /// Application start time. + /// </summary> + public long StartedTime { get; private set; } + + /// <summary> + /// Application finished time. + /// </summary> + public long FinishedTime { get; private set; } + + /// <summary> + /// Number of running evaluators + /// </summary> + public int NumberOfRunningEvaluators { get; private set; } + + /// <summary> + /// Driver url + /// </summary> + public Uri TrackingUrl { get; private set; } + + /// <summary> + /// Application id + /// </summary> + public string AppId { get; private set; } + + /// <summary> + /// Application final state + /// </summary> + public FinalState FinalState { get; private set; } + + /// <summary> + /// Application report constructor + /// </summary> + internal ApplicationReport(string appId, string appName, string trackingUrl, + long startedTime, long finishedTime, int numberOfRunningEvaluators, FinalState finalState) + { + AppId = appId; + AppName = appName; + TrackingUrl = trackingUrl == null ? null : new Uri(trackingUrl); + FinalState = finalState; + StartedTime = startedTime; + FinishedTime = finishedTime; + NumberOfRunningEvaluators = numberOfRunningEvaluators; + } + + /// <summary> + /// To string for printing/log + /// </summary> + /// <returns></returns> + public override string ToString() + { + return string.Format("AppName: {0} StartedTime: {1}, FinishedTime: {2}," + + "NumberOfRunningEvaluators: {3}, TrackingUrl: {4}, AppId: {5}, FinalState: {6}", + AppName, + StartedTime, + FinishedTime, + NumberOfRunningEvaluators, + TrackingUrl, + AppId, + FinalState); + } + } +} http://git-wip-us.apache.org/repos/asf/reef/blob/47593a77/lang/cs/Org.Apache.REEF.Client/YARN/IApplicationReport.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/IApplicationReport.cs b/lang/cs/Org.Apache.REEF.Client/YARN/IApplicationReport.cs new file mode 100644 index 0000000..66bfe14 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Client/YARN/IApplicationReport.cs @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System; +using Org.Apache.REEF.Client.YARN.RestClient.DataModel; +using Org.Apache.REEF.Utilities.Attributes; + +namespace Org.Apache.REEF.Client.YARN +{ + /// <summary> + /// This interface represents application information maintained by YARN RM. + /// This interface is modeled on Org.Apache.REEF.Client.YARN.RestClient.DataModel.Application. + /// Documentation on the class Org.Apache.REEF.Client.YARN.RestClient.DataModel.Application + /// can be found here. + /// <a href="http://hadoop.apache.org/docs/r2.6.0/hadoop-yarn/hadoop-yarn-site/WebServicesIntro.html";> + /// Hadoop RM REST API</a> documentation. + /// </summary> + [Unstable("0.17", "Working in progress. For local runtime, some of the property, such as FinalState and AppId are not implemented yet.")] + public interface IApplicationReport + { + /// <summary> + /// Get YARN application name. + /// </summary> + string AppName { get; } + + /// <summary> + /// Get YARN application start time. + /// </summary> + long StartedTime { get; } + + /// <summary> + /// Get YARN application finish time. + /// </summary> + long FinishedTime { get; } + + /// <summary> + /// Get YARN application number of running containers. + /// </summary> + int NumberOfRunningEvaluators { get; } + + /// <summary> + /// This method returns the url of http server running inside the driver. + /// e.g. http://hostname:port/ + /// </summary> + Uri TrackingUrl { get; } + + /// <summary> + /// Get YARN application id. + /// </summary> + string AppId { get; } + + /// <summary> + /// Get Application final state. + /// </summary> + FinalState FinalState { get; } + } +} http://git-wip-us.apache.org/repos/asf/reef/blob/47593a77/lang/cs/Org.Apache.REEF.Client/YARN/IYarnREEFClient.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/IYarnREEFClient.cs b/lang/cs/Org.Apache.REEF.Client/YARN/IYarnREEFClient.cs new file mode 100644 index 0000000..e208277 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Client/YARN/IYarnREEFClient.cs @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Collections.Generic; +using System.Threading.Tasks; +using Org.Apache.REEF.Client.API; +using Org.Apache.REEF.Utilities.Attributes; + +namespace Org.Apache.REEF.Client.YARN +{ + /// <summary> + /// Interface that defines Yarn client API + /// </summary> + public interface IYarnREEFClient : IREEFClient + { + /// <summary> + /// Returns all the application reports running in the cluster + /// </summary> + /// <returns></returns> + [Unstable("0.17", "Working in progress for rest API id returned")] + Task<IReadOnlyDictionary<string, IApplicationReport>> GetApplicationReports(); + } +} http://git-wip-us.apache.org/repos/asf/reef/blob/47593a77/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/DataModel/Application.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/DataModel/Application.cs b/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/DataModel/Application.cs index db7b0c7..ff09140 100644 --- a/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/DataModel/Application.cs +++ b/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/DataModel/Application.cs @@ -94,5 +94,38 @@ namespace Org.Apache.REEF.Client.YARN.RestClient.DataModel [JsonProperty("vcoreSeconds")] public long VcoreSeconds { get; set; } + + public override string ToString() + { + return string.Format("Id: {0}, User: {1}, Name: {2}, ApplicationType: {3}, " + + "Queue: {4}, State: {5}, FinalStatus: {6}, Progress: {7}, " + + "ClusterId: {8}, StartedTime: {9}, FinishedTime: {10}, " + + "ElapsedTime: {11}, AmContainerLogs: {12}, AmHostHttpAddress: {13}, " + + "AllocatedMB: {14}, AllocatedVCores: {15}, RunningContainers: {16}, " + + "MemorySeconds: {17}, VcoreSeconds: {18}, TrackingUI: {19}, " + + "TrackingUrl: {20}, Diagnostics: {21} ", + Id, + User, + Name, + ApplicationType, + Queue, + State, + FinalStatus, + Progress, + ClusterId, + StartedTime, + FinishedTime, + ElapsedTime, + AmContainerLogs, + AmHostHttpAddress, + AllocatedMB, + AllocatedVCores, + RunningContainers, + MemorySeconds, + VcoreSeconds, + TrackingUI, + TrackingUrl, + Diagnostics); + } } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/reef/blob/47593a77/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/IYarnRMClient.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/IYarnRMClient.cs b/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/IYarnRMClient.cs index 6a69374..ee63b60 100644 --- a/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/IYarnRMClient.cs +++ b/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/IYarnRMClient.cs @@ -37,6 +37,22 @@ namespace Org.Apache.REEF.Client.Yarn.RestClient Task<Application> GetApplicationAsync(string appId, CancellationToken cancellationToken); + /// <summary> + /// This API returns information about all the applications maintained + /// by YARN RM in the cluster by invoking REST API. + /// </summary> + /// <returns></returns> + Task<Applications> GetApplicationsAsync(); + + /// <summary> + /// This API returns information about all the applications maintained + /// by YARN RM in the cluster by invoking REST API. This API also allow cooperative + /// cancellation in multi-threading scenarios. + /// </summary> + /// <param name="cancellationToken"></param> + /// <returns></returns> + Task<Applications> GetApplicationsAsync(CancellationToken cancellationToken); + Task<NewApplication> CreateNewApplicationAsync(); Task<NewApplication> CreateNewApplicationAsync(CancellationToken cancellationToken); http://git-wip-us.apache.org/repos/asf/reef/blob/47593a77/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/YarnClient.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/YarnClient.cs b/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/YarnClient.cs index ec47d8b..6a29c49 100644 --- a/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/YarnClient.cs +++ b/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/YarnClient.cs @@ -91,6 +91,25 @@ namespace Org.Apache.REEF.Client.Yarn.RestClient await GenerateUrlAndExecuteRequestAsync<Application>(request, cancellationToken); } + /// <summary> + /// This API returns information about all the applications maintained + /// by YARN RM in the cluster by invoking REST API. This API also allow cooperative + /// cancellation in multi-threading scenarios. + /// </summary> + /// <param name="cancellationToken">cancellation token</param> + /// <returns>list of applications</returns> + public async Task<Applications> GetApplicationsAsync(CancellationToken cancellationToken) + { + await new RemoveSynchronizationContextAwaiter(); + + var request = _requestFactory.CreateRestRequest(Applications.Resource, + Method.GET, + Applications.RootElement); + + return + await GenerateUrlAndExecuteRequestAsync<Applications>(request, cancellationToken); + } + public async Task<NewApplication> CreateNewApplicationAsync(CancellationToken cancellationToken) { await new RemoveSynchronizationContextAwaiter(); http://git-wip-us.apache.org/repos/asf/reef/blob/47593a77/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/YarnClientNoCancellationToken.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/YarnClientNoCancellationToken.cs b/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/YarnClientNoCancellationToken.cs index 707a3a0..2d490f8 100644 --- a/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/YarnClientNoCancellationToken.cs +++ b/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/YarnClientNoCancellationToken.cs @@ -24,30 +24,55 @@ namespace Org.Apache.REEF.Client.Yarn.RestClient { internal sealed partial class YarnClient { + /// <summary> + /// Get Cluster Info without cancellation token. + /// </summary> public async Task<ClusterInfo> GetClusterInfoAsync() { await new RemoveSynchronizationContextAwaiter(); return await GetClusterInfoAsync(CancellationToken.None); } + /// <summary> + /// Get Cluster Metrics without cancellation token. + /// </summary> public async Task<ClusterMetrics> GetClusterMetricsAsync() { await new RemoveSynchronizationContextAwaiter(); return await GetClusterMetricsAsync(CancellationToken.None); } + /// <summary> + /// Get Application without cancellation token. + /// </summary> public async Task<Application> GetApplicationAsync(string appId) { await new RemoveSynchronizationContextAwaiter(); return await GetApplicationAsync(appId, CancellationToken.None); } + /// <summary> + /// Get Applications without cancellation token. + /// </summary> + public async Task<Applications> GetApplicationsAsync() + { + await new RemoveSynchronizationContextAwaiter(); + return await GetApplicationsAsync(CancellationToken.None); + } + + /// <summary> + /// Create a new Application without cancellation token. + /// </summary> public async Task<NewApplication> CreateNewApplicationAsync() { await new RemoveSynchronizationContextAwaiter(); return await CreateNewApplicationAsync(CancellationToken.None); } + /// <summary> + /// Submit Application without cancellation token. + /// </summary> + /// <param name="submitApplicationRequest"></param> public async Task<Application> SubmitApplicationAsync(SubmitApplication submitApplicationRequest) { await new RemoveSynchronizationContextAwaiter(); http://git-wip-us.apache.org/repos/asf/reef/blob/47593a77/lang/cs/Org.Apache.REEF.Client/YARN/YARNClientConfiguration.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/YARNClientConfiguration.cs b/lang/cs/Org.Apache.REEF.Client/YARN/YARNClientConfiguration.cs index 20127c6..69d715e 100644 --- a/lang/cs/Org.Apache.REEF.Client/YARN/YARNClientConfiguration.cs +++ b/lang/cs/Org.Apache.REEF.Client/YARN/YARNClientConfiguration.cs @@ -43,6 +43,7 @@ namespace Org.Apache.REEF.Client.Yarn public static readonly OptionalParameter<string> FileSystemUrl = new OptionalParameter<string>(); public static ConfigurationModule ConfigurationModule = new YARNClientConfiguration() + .BindImplementation(GenericType<IYarnREEFClient>.Class, GenericType<YarnREEFClient>.Class) .BindImplementation(GenericType<IREEFClient>.Class, GenericType<YarnREEFClient>.Class) .BindImplementation(GenericType<IYarnRestClientCredential>.Class, YarnRestClientCredential) .BindNamedParameter(GenericType<JobSubmissionDirectoryPrefixParameter>.Class, JobSubmissionFolderPrefix) http://git-wip-us.apache.org/repos/asf/reef/blob/47593a77/lang/cs/Org.Apache.REEF.Client/YARN/YARNREEFClient.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/YARNREEFClient.cs b/lang/cs/Org.Apache.REEF.Client/YARN/YARNREEFClient.cs index 20c8389..c00be21 100644 --- a/lang/cs/Org.Apache.REEF.Client/YARN/YARNREEFClient.cs +++ b/lang/cs/Org.Apache.REEF.Client/YARN/YARNREEFClient.cs @@ -16,6 +16,8 @@ // under the License. using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; using System.Globalization; using System.IO; using System.Linq; @@ -32,7 +34,7 @@ using Org.Apache.REEF.Utilities.Logging; namespace Org.Apache.REEF.Client.Yarn { - internal sealed class YarnREEFClient : IREEFClient + internal sealed class YarnREEFClient : IYarnREEFClient { /// <summary> /// The class name that contains the Java counterpart for this client. @@ -98,13 +100,50 @@ namespace Org.Apache.REEF.Client.Yarn { var application = await _yarnClient.GetApplicationAsync(appId); - var msg = string.Format("application status {0}, Progress: {1}, trackingUri: {2}, Name: {3}, ApplicationId: {4}, State {5}.", - application.FinalStatus, application.Progress, application.TrackingUI, application.Name, application.Id, application.State); - Logger.Log(Level.Verbose, msg); + Logger.Log(Level.Verbose, + "application status {0}, Progress: {1}, uri: {2}, Name: {3}, ApplicationId: {4}, State {5}.", + application.FinalStatus, + application.Progress, + application.TrackingUI, + application.Name, + application.Id, + application.State); return application.FinalStatus; } + /// <summary> + /// Returns all the application reports running in the cluster. + /// GetApplicationReports call is very expensive as it is trying + /// fetch information about all the applications in the cluster. + /// + /// If this method is called right after submitting a new app then + /// that new app might not immediately result in this list until + /// some number of retries. + /// </summary> + /// <returns></returns> + public async Task<IReadOnlyDictionary<string, IApplicationReport>> GetApplicationReports() + { + var appReports = new Dictionary<string, IApplicationReport>(); + var applications = await _yarnClient.GetApplicationsAsync(); + + foreach (var application in applications.App) + { + appReports.Add(application.Id, new ApplicationReport(application.Id, + application.Name, + application.TrackingUrl, + application.StartedTime, + application.FinishedTime, + application.RunningContainers, + application.FinalStatus)); + + Logger.Log(Level.Verbose, + "Application report {0}: {1}", + application.Id, application); + } + return new ReadOnlyDictionary<string, IApplicationReport>(appReports); + } + private void Launch(JobRequest jobRequest, string driverFolderPath) { _driverFolderPreparationHelper.PrepareDriverFolder(jobRequest.AppParameters, driverFolderPath); http://git-wip-us.apache.org/repos/asf/reef/blob/47593a77/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFDotNetClient.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFDotNetClient.cs b/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFDotNetClient.cs index 18f8fbf..fcae2e4 100644 --- a/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFDotNetClient.cs +++ b/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFDotNetClient.cs @@ -40,7 +40,7 @@ namespace Org.Apache.REEF.Client.YARN /// deprecated by final client. /// </summary> [Unstable("For security token support we still need to use YARNREEFClient until (REEF-875)")] - public sealed class YarnREEFDotNetClient : IREEFClient + public sealed class YarnREEFDotNetClient : IYarnREEFClient { private const string REEFApplicationType = @"REEF"; private static readonly Logger Log = Logger.GetLogger(typeof(YarnREEFDotNetClient)); @@ -146,6 +146,17 @@ namespace Org.Apache.REEF.Client.YARN return application.FinalStatus; } + /// <summary> + /// Returns all the application reports running in the cluster + /// </summary> + /// <returns></returns> + /// TODO: [REEF-1825]: Implement GetApplicationReports to return the status of the applications. + public async Task<IReadOnlyDictionary<string, IApplicationReport>> GetApplicationReports() + { + await Task.Delay(0); + throw new NotImplementedException(); + } + private SubmitApplication CreateApplicationSubmissionRequest( JobParameters jobParameters, string appId, http://git-wip-us.apache.org/repos/asf/reef/blob/47593a77/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEFYarn.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEFYarn.cs b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEFYarn.cs index fbccdec..6044569 100644 --- a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEFYarn.cs +++ b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEFYarn.cs @@ -22,6 +22,7 @@ using System.Threading; using Org.Apache.REEF.Client.API; using Org.Apache.REEF.Client.Common; using Org.Apache.REEF.Client.Yarn; +using Org.Apache.REEF.Client.YARN; using Org.Apache.REEF.Client.YARN.RestClient.DataModel; using Org.Apache.REEF.Driver; using Org.Apache.REEF.Tang.Annotations; @@ -46,7 +47,7 @@ namespace Org.Apache.REEF.Examples.HelloREEF private const string SecurityTokenId = "SecurityTokenId"; private const string SecurityTokenPwd = "SecurityTokenPwd"; - private readonly IREEFClient _reefClient; + private readonly IYarnREEFClient _reefClient; private readonly JobRequestBuilder _jobRequestBuilder; private static readonly Logger Logger = Logger.GetLogger(typeof(HelloREEFYarn)); @@ -57,7 +58,7 @@ namespace Org.Apache.REEF.Examples.HelloREEF private readonly IList<string> _nodeNames; [Inject] - private HelloREEFYarn(IREEFClient reefClient, + private HelloREEFYarn(IYarnREEFClient reefClient, JobRequestBuilder jobRequestBuilder, [Parameter(typeof(NodeNames))] ISet<string> nodeNames) { @@ -94,11 +95,26 @@ namespace Org.Apache.REEF.Examples.HelloREEF .Build(); var result = _reefClient.SubmitAndGetJobStatus(helloJobRequest); + + LogApplicationReport(); var state = PullFinalJobStatus(result); Logger.Log(Level.Info, "Application state : {0}.", state); } /// <summary> + /// Get application report and log. + /// </summary> + private void LogApplicationReport() + { + Logger.Log(Level.Info, "Getting Application report..."); + var apps = _reefClient.GetApplicationReports().Result; + foreach (var r in apps) + { + Logger.Log(Level.Info, "Application report -- AppId {0}: {1}.", r.Key, r.Value.ToString()); + } + } + + /// <summary> /// This is to pull job final status until the Job is done /// </summary> /// <param name="jobSubmitionResult"></param>
