Repository: reef
Updated Branches:
  refs/heads/master 721f8ca48 -> 47593a770


[REEF-1823] Support YARN API GetApplications() in .Net YarnREEFClient

This addressed the issue by
  * Exposing GetApplications in IYarnRMClient
  * Adding a IYarnREEFClient for Yarn specific functions
  * Implementing the new functionality in IYarnREEFClient
  * Updating YARNClientConfiguration to bind IYarnREEFClient to YarnREEFClient
  * Continuing to bind IREEFClient also to YarnREEFClient for backward 
compatibility
  * Adding test cases
  * Updating HelloREEFYarn to use the new API

JIRA:
  [REEF-1823](https://issues.apache.org/jira/browse/REEF-1823)

Pull request:
  This closes #1361


Project: http://git-wip-us.apache.org/repos/asf/reef/repo
Commit: http://git-wip-us.apache.org/repos/asf/reef/commit/47593a77
Tree: http://git-wip-us.apache.org/repos/asf/reef/tree/47593a77
Diff: http://git-wip-us.apache.org/repos/asf/reef/diff/47593a77

Branch: refs/heads/master
Commit: 47593a77059e082b1e094af3b6811473cb5ac628
Parents: 721f8ca
Author: Julia Wang <jul...@apache.org>
Authored: Tue Aug 8 23:17:59 2017 -0700
Committer: Shravan Narayanamurthy <shra...@apache.org>
Committed: Wed Aug 9 18:23:03 2017 -0700

----------------------------------------------------------------------
 .../YarnClientTests.cs                          |  63 ++++++++++++
 .../Org.Apache.REEF.Client.csproj               |   3 +
 .../YARN/ApplicationReport.cs                   | 101 +++++++++++++++++++
 .../YARN/IApplicationReport.cs                  |  71 +++++++++++++
 .../YARN/IYarnREEFClient.cs                     |  37 +++++++
 .../YARN/RESTClient/DataModel/Application.cs    |  33 ++++++
 .../YARN/RESTClient/IYarnRMClient.cs            |  16 +++
 .../YARN/RESTClient/YarnClient.cs               |  19 ++++
 .../RESTClient/YarnClientNoCancellationToken.cs |  25 +++++
 .../YARN/YARNClientConfiguration.cs             |   1 +
 .../YARN/YARNREEFClient.cs                      |  47 ++++++++-
 .../YARN/YarnREEFDotNetClient.cs                |  13 ++-
 .../HelloREEFYarn.cs                            |  20 +++-
 13 files changed, 442 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/reef/blob/47593a77/lang/cs/Org.Apache.REEF.Client.Tests/YarnClientTests.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client.Tests/YarnClientTests.cs 
b/lang/cs/Org.Apache.REEF.Client.Tests/YarnClientTests.cs
index 3858b12..127ce52 100644
--- a/lang/cs/Org.Apache.REEF.Client.Tests/YarnClientTests.cs
+++ b/lang/cs/Org.Apache.REEF.Client.Tests/YarnClientTests.cs
@@ -132,6 +132,69 @@ namespace Org.Apache.REEF.Client.Tests
         }
 
         [Fact]
+        public async Task TestGetApplications()
+        {
+            var ctx = new TestContext();
+            var urlProvider = ctx.UrlProviderFake;
+            var restReqExecutor = ctx.RestRequestExecutorFake;
+            var anyUri = Enumerable.Repeat(new Uri("anyscheme://anypath"), 1);
+
+            urlProvider.GetUrlAsync().Returns(Task.FromResult(anyUri));
+            var anyApplications = new Applications();
+
+            var anyApplication1 = new Application
+            {
+                AllocatedMB = 100,
+                AmHostHttpAddress = "http://anyhttpaddress";,
+                AmContainerLogs = "SomeLogs",
+                ApplicationType = "AnyYarnApplicationType",
+                State = State.FINISHED,
+                Name = "AnyApplicationName",
+                RunningContainers = 0
+            };
+
+            var anyApplication2 = new Application
+            {
+                AllocatedMB = 100,
+                AmHostHttpAddress = "http://anyhttpaddress";,
+                AmContainerLogs = "SomeLogs",
+                ApplicationType = "AnyYarnApplicationType",
+                State = State.FINISHED,
+                Name = "AnyApplicationName",
+                RunningContainers = 0
+            };
+
+            anyApplications.App = new List<Application> { anyApplication1, 
anyApplication2 };
+
+            restReqExecutor.ExecuteAsync<Applications>(
+                Arg.Is<RestRequest>(
+                    req =>
+                        req.Resource == "ws/v1/cluster/apps"
+                        && req.RootElement == Applications.RootElement
+                        && req.Method == Method.GET),
+                anyUri.First(),
+                
CancellationToken.None).Returns(Task.FromResult(anyApplications));
+
+            var yarnClient = ctx.GetClient();
+            Applications actualApplications = await 
yarnClient.GetApplicationsAsync();
+
+            Assert.NotEqual(actualApplications.App, null);
+            Assert.Equal(actualApplications.App.Count, 2);
+
+            int matchCount = 0;
+            foreach (var anyApplication in actualApplications.App)
+            {
+                if (anyApplication == anyApplication1 || anyApplication == 
anyApplication2)
+                {
+                    ++matchCount;
+                }
+            }
+
+            Assert.Equal(actualApplications.App.Count, matchCount);
+            var unused = urlProvider.Received(1).GetUrlAsync();
+        }
+
+        [Fact]
         public async Task TestGetApplicationFinalStatus()
         {
             var ctx = new TestContext();

http://git-wip-us.apache.org/repos/asf/reef/blob/47593a77/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj 
b/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj
index c8df560..2c3df86 100644
--- a/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj
+++ b/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj
@@ -99,18 +99,21 @@ under the License.
     <Compile Include="Local\Parameters\LocalRuntimeDirectory.cs" />
     <Compile Include="Local\Parameters\NumberOfEvaluators.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="YARN\ApplicationReport.cs" />
     <Compile Include="YARN\HDI\HDInsightClientConfiguration.cs" />
     <Compile Include="YARN\HDI\HDInsightCommandLineEnvironment.cs" />
     <Compile Include="YARN\HDI\HDInsightCredential.cs" />
     <Compile 
Include="YARN\HDI\HDInsightResourceFileRemoteUrlToClusterUrlConverter.cs" />
     <Compile Include="YARN\HDI\HDInsightUrlProvider.cs" />
     <Compile Include="YARN\HDI\NamedParameters.cs" />
+    <Compile Include="YARN\IApplicationReport.cs" />
     <Compile 
Include="YARN\IdentityResourceFileRemoteUrlToClusterUrlConverter.cs" />
     <Compile Include="YARN\IResourceFileRemoteUrlToClusterUrlConverter.cs" />
     <Compile Include="YARN\IYarnCommandLineEnvironment.cs" />
     <Compile Include="YARN\IYarnJobCommandProvider.cs" />
     <Compile Include="YARN\IJobResourceUploader.cs" />
     <Compile Include="YARN\IJobSubmissionDirectoryProvider.cs" />
+    <Compile Include="YARN\IYarnREEFClient.cs" />
     <Compile Include="YARN\Parameters\DriverMaxMemoryAllicationPoolSizeMB.cs" 
/>
     <Compile Include="YARN\Parameters\DriverMaxPermSizeMB.cs" />
     <Compile Include="YARN\Parameters\DriverStderrFilePath.cs" />

http://git-wip-us.apache.org/repos/asf/reef/blob/47593a77/lang/cs/Org.Apache.REEF.Client/YARN/ApplicationReport.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/ApplicationReport.cs 
b/lang/cs/Org.Apache.REEF.Client/YARN/ApplicationReport.cs
new file mode 100644
index 0000000..6cbd89c
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Client/YARN/ApplicationReport.cs
@@ -0,0 +1,101 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System;
+using Org.Apache.REEF.Client.YARN.RestClient.DataModel;
+using Org.Apache.REEF.Utilities.Logging;
+
+namespace Org.Apache.REEF.Client.YARN
+{
+    /// <summary>
+    /// This class represents application information maintained by YARN RM.
+    /// This class is modeled on 
Org.Apache.REEF.Client.YARN.RestClient.DataModel.Application. 
+    /// Documentation on the class 
Org.Apache.REEF.Client.YARN.RestClient.DataModel.Application
+    /// can be found here.
+    /// <a 
href="http://hadoop.apache.org/docs/r2.6.0/hadoop-yarn/hadoop-yarn-site/WebServicesIntro.html";>
+    /// Hadoop RM REST API</a> documentation.
+    /// </summary>
+    internal sealed class ApplicationReport : IApplicationReport
+    {
+        /// <summary>
+        /// Application name.
+        /// </summary>
+        public string AppName { get; private set; }
+
+        /// <summary>
+        /// Application start time.
+        /// </summary>
+        public long StartedTime { get; private set; }
+
+        /// <summary>
+        /// Application finished time.
+        /// </summary>
+        public long FinishedTime { get; private set; }
+
+        /// <summary>
+        /// Number of running evaluators
+        /// </summary>
+        public int NumberOfRunningEvaluators { get; private set; }
+
+        /// <summary>
+        /// Driver url
+        /// </summary>
+        public Uri TrackingUrl { get; private set; }
+
+        /// <summary>
+        /// Application id
+        /// </summary>
+        public string AppId { get; private set; }
+
+        /// <summary>
+        /// Application final state
+        /// </summary>
+        public FinalState FinalState { get; private set; }
+
+        /// <summary>
+        /// Application report constructor
+        /// </summary>
+        internal ApplicationReport(string appId, string appName, string 
trackingUrl, 
+            long startedTime, long finishedTime, int 
numberOfRunningEvaluators, FinalState finalState)
+        {
+            AppId = appId;
+            AppName = appName;
+            TrackingUrl = trackingUrl == null ? null : new Uri(trackingUrl);
+            FinalState = finalState;
+            StartedTime = startedTime;
+            FinishedTime = finishedTime;
+            NumberOfRunningEvaluators = numberOfRunningEvaluators;
+        }
+
+        /// <summary>
+        /// To string for printing/log
+        /// </summary>
+        /// <returns></returns>
+        public override string ToString()
+        {
+            return string.Format("AppName: {0} StartedTime: {1}, FinishedTime: 
{2}," +
+                                    "NumberOfRunningEvaluators: {3}, 
TrackingUrl: {4}, AppId: {5}, FinalState: {6}",
+                AppName,
+                StartedTime,
+                FinishedTime,
+                NumberOfRunningEvaluators,
+                TrackingUrl,
+                AppId,
+                FinalState);
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/reef/blob/47593a77/lang/cs/Org.Apache.REEF.Client/YARN/IApplicationReport.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/IApplicationReport.cs 
b/lang/cs/Org.Apache.REEF.Client/YARN/IApplicationReport.cs
new file mode 100644
index 0000000..66bfe14
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Client/YARN/IApplicationReport.cs
@@ -0,0 +1,71 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System;
+using Org.Apache.REEF.Client.YARN.RestClient.DataModel;
+using Org.Apache.REEF.Utilities.Attributes;
+
+namespace Org.Apache.REEF.Client.YARN
+{
+    /// <summary>
+    /// This interface represents application information maintained by YARN 
RM.
+    /// This interface is modeled on 
Org.Apache.REEF.Client.YARN.RestClient.DataModel.Application. 
+    /// Documentation on the class 
Org.Apache.REEF.Client.YARN.RestClient.DataModel.Application
+    /// can be found here.
+    /// <a 
href="http://hadoop.apache.org/docs/r2.6.0/hadoop-yarn/hadoop-yarn-site/WebServicesIntro.html";>
+    /// Hadoop RM REST API</a> documentation.
+    /// </summary>
+    [Unstable("0.17", "Working in progress. For local runtime, some of the 
property, such as FinalState and AppId are not implemented yet.")]
+    public interface IApplicationReport
+    {
+        /// <summary>
+        /// Get YARN application name.
+        /// </summary>
+        string AppName { get; }
+        
+        /// <summary>
+        /// Get YARN application start time.
+        /// </summary>
+        long StartedTime { get; }
+
+        /// <summary>
+        /// Get YARN application finish time.
+        /// </summary>
+        long FinishedTime { get; }
+
+        /// <summary>
+        /// Get YARN application number of running containers.
+        /// </summary>
+        int NumberOfRunningEvaluators { get; }
+
+        /// <summary>
+        /// This method returns the url of http server running inside the 
driver.
+        /// e.g. http://hostname:port/
+        /// </summary>
+        Uri TrackingUrl { get; }
+
+        /// <summary>
+        /// Get YARN application id.
+        /// </summary>
+        string AppId { get; }
+
+        /// <summary>
+        /// Get Application final state.
+        /// </summary>
+        FinalState FinalState { get; }
+    }
+}

http://git-wip-us.apache.org/repos/asf/reef/blob/47593a77/lang/cs/Org.Apache.REEF.Client/YARN/IYarnREEFClient.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/IYarnREEFClient.cs 
b/lang/cs/Org.Apache.REEF.Client/YARN/IYarnREEFClient.cs
new file mode 100644
index 0000000..e208277
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Client/YARN/IYarnREEFClient.cs
@@ -0,0 +1,37 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using Org.Apache.REEF.Client.API;
+using Org.Apache.REEF.Utilities.Attributes;
+
+namespace Org.Apache.REEF.Client.YARN
+{
+    /// <summary>
+    /// Interface that defines Yarn client API
+    /// </summary>
+    public interface IYarnREEFClient : IREEFClient
+    {
+        /// <summary>
+        /// Returns all the application reports running in the cluster
+        /// </summary>
+        /// <returns></returns>
+        [Unstable("0.17", "Working in progress for rest API id returned")]
+        Task<IReadOnlyDictionary<string, IApplicationReport>> 
GetApplicationReports();
+    }
+}

http://git-wip-us.apache.org/repos/asf/reef/blob/47593a77/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/DataModel/Application.cs
----------------------------------------------------------------------
diff --git 
a/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/DataModel/Application.cs 
b/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/DataModel/Application.cs
index db7b0c7..ff09140 100644
--- a/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/DataModel/Application.cs
+++ b/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/DataModel/Application.cs
@@ -94,5 +94,38 @@ namespace Org.Apache.REEF.Client.YARN.RestClient.DataModel
 
         [JsonProperty("vcoreSeconds")]
         public long VcoreSeconds { get; set; }
+
+        public override string ToString()
+        {
+            return string.Format("Id: {0}, User: {1}, Name: {2}, 
ApplicationType: {3}, " +
+                                 "Queue: {4}, State: {5}, FinalStatus: {6}, 
Progress: {7}, " +
+                                 "ClusterId: {8}, StartedTime: {9}, 
FinishedTime: {10}, " +
+                                 "ElapsedTime: {11}, AmContainerLogs: {12}, 
AmHostHttpAddress: {13}, " +
+                                 "AllocatedMB: {14}, AllocatedVCores: {15}, 
RunningContainers: {16}, " +
+                                 "MemorySeconds: {17}, VcoreSeconds: {18}, 
TrackingUI: {19}, " +
+                                 "TrackingUrl: {20}, Diagnostics: {21} ",
+                Id,
+                User,
+                Name,
+                ApplicationType,
+                Queue,
+                State,
+                FinalStatus,
+                Progress,
+                ClusterId,
+                StartedTime,
+                FinishedTime,
+                ElapsedTime,
+                AmContainerLogs,
+                AmHostHttpAddress,
+                AllocatedMB,
+                AllocatedVCores,
+                RunningContainers,
+                MemorySeconds,
+                VcoreSeconds,
+                TrackingUI,
+                TrackingUrl,
+                Diagnostics);
+        }
     }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/reef/blob/47593a77/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/IYarnRMClient.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/IYarnRMClient.cs 
b/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/IYarnRMClient.cs
index 6a69374..ee63b60 100644
--- a/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/IYarnRMClient.cs
+++ b/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/IYarnRMClient.cs
@@ -37,6 +37,22 @@ namespace Org.Apache.REEF.Client.Yarn.RestClient
 
         Task<Application> GetApplicationAsync(string appId, CancellationToken 
cancellationToken);
 
+        /// <summary>
+        /// This API returns information about all the applications maintained
+        /// by YARN RM in the cluster by invoking REST API.
+        /// </summary>
+        /// <returns></returns>
+        Task<Applications> GetApplicationsAsync();
+
+        /// <summary>
+        /// This API returns information about all the applications maintained
+        /// by YARN RM in the cluster by invoking REST API. This API also 
allow cooperative
+        /// cancellation in multi-threading scenarios.
+        /// </summary>
+        /// <param name="cancellationToken"></param>
+        /// <returns></returns>
+        Task<Applications> GetApplicationsAsync(CancellationToken 
cancellationToken);
+
         Task<NewApplication> CreateNewApplicationAsync();
 
         Task<NewApplication> CreateNewApplicationAsync(CancellationToken 
cancellationToken);

http://git-wip-us.apache.org/repos/asf/reef/blob/47593a77/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/YarnClient.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/YarnClient.cs 
b/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/YarnClient.cs
index ec47d8b..6a29c49 100644
--- a/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/YarnClient.cs
+++ b/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/YarnClient.cs
@@ -91,6 +91,25 @@ namespace Org.Apache.REEF.Client.Yarn.RestClient
                 await GenerateUrlAndExecuteRequestAsync<Application>(request, 
cancellationToken);
         }
 
+        /// <summary>
+        /// This API returns information about all the applications maintained
+        /// by YARN RM in the cluster by invoking REST API. This API also 
allow cooperative
+        /// cancellation in multi-threading scenarios.
+        /// </summary>
+        /// <param name="cancellationToken">cancellation token</param>
+        /// <returns>list of applications</returns>
+        public async Task<Applications> GetApplicationsAsync(CancellationToken 
cancellationToken)
+        {
+            await new RemoveSynchronizationContextAwaiter();
+
+            var request = 
_requestFactory.CreateRestRequest(Applications.Resource,
+                Method.GET,
+                Applications.RootElement);
+
+            return
+                await GenerateUrlAndExecuteRequestAsync<Applications>(request, 
cancellationToken);
+        }
+
         public async Task<NewApplication> 
CreateNewApplicationAsync(CancellationToken cancellationToken)
         {
             await new RemoveSynchronizationContextAwaiter();

http://git-wip-us.apache.org/repos/asf/reef/blob/47593a77/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/YarnClientNoCancellationToken.cs
----------------------------------------------------------------------
diff --git 
a/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/YarnClientNoCancellationToken.cs
 
b/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/YarnClientNoCancellationToken.cs
index 707a3a0..2d490f8 100644
--- 
a/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/YarnClientNoCancellationToken.cs
+++ 
b/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/YarnClientNoCancellationToken.cs
@@ -24,30 +24,55 @@ namespace Org.Apache.REEF.Client.Yarn.RestClient
 {
     internal sealed partial class YarnClient
     {
+        /// <summary>
+        /// Get Cluster Info without cancellation token.
+        /// </summary>
         public async Task<ClusterInfo> GetClusterInfoAsync()
         {
             await new RemoveSynchronizationContextAwaiter();
             return await GetClusterInfoAsync(CancellationToken.None);
         }
 
+        /// <summary>
+        /// Get Cluster Metrics without cancellation token.
+        /// </summary>
         public async Task<ClusterMetrics> GetClusterMetricsAsync()
         {
             await new RemoveSynchronizationContextAwaiter();
             return await GetClusterMetricsAsync(CancellationToken.None);
         }
 
+        /// <summary>
+        /// Get Application without cancellation token.
+        /// </summary>
         public async Task<Application> GetApplicationAsync(string appId)
         {
             await new RemoveSynchronizationContextAwaiter();
             return await GetApplicationAsync(appId, CancellationToken.None);
         }
 
+        /// <summary>
+        /// Get Applications without cancellation token.
+        /// </summary>
+        public async Task<Applications> GetApplicationsAsync()
+        {
+            await new RemoveSynchronizationContextAwaiter();
+            return await GetApplicationsAsync(CancellationToken.None);
+        }
+
+        /// <summary>
+        /// Create a new Application without cancellation token.
+        /// </summary>
         public async Task<NewApplication> CreateNewApplicationAsync()
         {
             await new RemoveSynchronizationContextAwaiter();
             return await CreateNewApplicationAsync(CancellationToken.None);
         }
 
+        /// <summary>
+        /// Submit Application without cancellation token.
+        /// </summary>
+        /// <param name="submitApplicationRequest"></param>
         public async Task<Application> 
SubmitApplicationAsync(SubmitApplication submitApplicationRequest)
         {
             await new RemoveSynchronizationContextAwaiter();

http://git-wip-us.apache.org/repos/asf/reef/blob/47593a77/lang/cs/Org.Apache.REEF.Client/YARN/YARNClientConfiguration.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/YARNClientConfiguration.cs 
b/lang/cs/Org.Apache.REEF.Client/YARN/YARNClientConfiguration.cs
index 20127c6..69d715e 100644
--- a/lang/cs/Org.Apache.REEF.Client/YARN/YARNClientConfiguration.cs
+++ b/lang/cs/Org.Apache.REEF.Client/YARN/YARNClientConfiguration.cs
@@ -43,6 +43,7 @@ namespace Org.Apache.REEF.Client.Yarn
         public static readonly OptionalParameter<string> FileSystemUrl = new 
OptionalParameter<string>();
 
         public static ConfigurationModule ConfigurationModule = new 
YARNClientConfiguration()
+            .BindImplementation(GenericType<IYarnREEFClient>.Class, 
GenericType<YarnREEFClient>.Class)
             .BindImplementation(GenericType<IREEFClient>.Class, 
GenericType<YarnREEFClient>.Class)
             .BindImplementation(GenericType<IYarnRestClientCredential>.Class, 
YarnRestClientCredential)
             
.BindNamedParameter(GenericType<JobSubmissionDirectoryPrefixParameter>.Class, 
JobSubmissionFolderPrefix)

http://git-wip-us.apache.org/repos/asf/reef/blob/47593a77/lang/cs/Org.Apache.REEF.Client/YARN/YARNREEFClient.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/YARNREEFClient.cs 
b/lang/cs/Org.Apache.REEF.Client/YARN/YARNREEFClient.cs
index 20c8389..c00be21 100644
--- a/lang/cs/Org.Apache.REEF.Client/YARN/YARNREEFClient.cs
+++ b/lang/cs/Org.Apache.REEF.Client/YARN/YARNREEFClient.cs
@@ -16,6 +16,8 @@
 // under the License.
 
 using System;
+using System.Collections.Generic;
+using System.Collections.ObjectModel;
 using System.Globalization;
 using System.IO;
 using System.Linq;
@@ -32,7 +34,7 @@ using Org.Apache.REEF.Utilities.Logging;
 
 namespace Org.Apache.REEF.Client.Yarn
 {
-    internal sealed class YarnREEFClient : IREEFClient
+    internal sealed class YarnREEFClient : IYarnREEFClient
     {
         /// <summary>
         /// The class name that contains the Java counterpart for this client.
@@ -98,13 +100,50 @@ namespace Org.Apache.REEF.Client.Yarn
         {
             var application = await _yarnClient.GetApplicationAsync(appId);
 
-            var msg = string.Format("application status {0}, Progress: {1}, 
trackingUri: {2}, Name: {3}, ApplicationId: {4}, State {5}.",
-                application.FinalStatus, application.Progress, 
application.TrackingUI, application.Name, application.Id, application.State);
-            Logger.Log(Level.Verbose, msg);
+            Logger.Log(Level.Verbose,
+               "application status {0}, Progress: {1}, uri: {2}, Name: {3}, 
ApplicationId: {4}, State {5}.",
+               application.FinalStatus,
+               application.Progress,
+               application.TrackingUI,
+               application.Name,
+               application.Id,
+               application.State);
 
             return application.FinalStatus;
         }
 
+        /// <summary>
+        /// Returns all the application reports running in the cluster.
+        /// GetApplicationReports call is very expensive as it is trying 
+        /// fetch information about all the applications in the cluster.
+        /// 
+        /// If this method is called right after submitting a new app then
+        /// that new app might not immediately result in this list until 
+        /// some number of retries. 
+        /// </summary>
+        /// <returns></returns>
+        public async Task<IReadOnlyDictionary<string, IApplicationReport>> 
GetApplicationReports()
+        {
+            var appReports = new Dictionary<string, IApplicationReport>();
+            var applications = await _yarnClient.GetApplicationsAsync();
+
+            foreach (var application in applications.App)
+            {
+                appReports.Add(application.Id, new 
ApplicationReport(application.Id,
+                    application.Name,
+                    application.TrackingUrl,
+                    application.StartedTime,
+                    application.FinishedTime,
+                    application.RunningContainers,
+                    application.FinalStatus));
+
+                Logger.Log(Level.Verbose,
+                    "Application report {0}: {1}",
+                    application.Id, application);
+            }
+            return new ReadOnlyDictionary<string, 
IApplicationReport>(appReports);
+        }
+
         private void Launch(JobRequest jobRequest, string driverFolderPath)
         {
             
_driverFolderPreparationHelper.PrepareDriverFolder(jobRequest.AppParameters, 
driverFolderPath);

http://git-wip-us.apache.org/repos/asf/reef/blob/47593a77/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFDotNetClient.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFDotNetClient.cs 
b/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFDotNetClient.cs
index 18f8fbf..fcae2e4 100644
--- a/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFDotNetClient.cs
+++ b/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFDotNetClient.cs
@@ -40,7 +40,7 @@ namespace Org.Apache.REEF.Client.YARN
     /// deprecated by final client.
     /// </summary>
     [Unstable("For security token support we still need to use YARNREEFClient 
until (REEF-875)")]
-    public sealed class YarnREEFDotNetClient : IREEFClient
+    public sealed class YarnREEFDotNetClient : IYarnREEFClient
     {
         private const string REEFApplicationType = @"REEF";
         private static readonly Logger Log = 
Logger.GetLogger(typeof(YarnREEFDotNetClient));
@@ -146,6 +146,17 @@ namespace Org.Apache.REEF.Client.YARN
             return application.FinalStatus;
         }
 
+        /// <summary>
+        /// Returns all the application reports running in the cluster
+        /// </summary>
+        /// <returns></returns>
+        /// TODO: [REEF-1825]: Implement GetApplicationReports to return the 
status of the applications.
+        public async Task<IReadOnlyDictionary<string, IApplicationReport>> 
GetApplicationReports()
+        {
+            await Task.Delay(0);
+            throw new NotImplementedException();
+        }
+
         private SubmitApplication CreateApplicationSubmissionRequest(
            JobParameters jobParameters,
            string appId,

http://git-wip-us.apache.org/repos/asf/reef/blob/47593a77/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEFYarn.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEFYarn.cs 
b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEFYarn.cs
index fbccdec..6044569 100644
--- a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEFYarn.cs
+++ b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEFYarn.cs
@@ -22,6 +22,7 @@ using System.Threading;
 using Org.Apache.REEF.Client.API;
 using Org.Apache.REEF.Client.Common;
 using Org.Apache.REEF.Client.Yarn;
+using Org.Apache.REEF.Client.YARN;
 using Org.Apache.REEF.Client.YARN.RestClient.DataModel;
 using Org.Apache.REEF.Driver;
 using Org.Apache.REEF.Tang.Annotations;
@@ -46,7 +47,7 @@ namespace Org.Apache.REEF.Examples.HelloREEF
         private const string SecurityTokenId = "SecurityTokenId";
         private const string SecurityTokenPwd = "SecurityTokenPwd";
 
-        private readonly IREEFClient _reefClient;
+        private readonly IYarnREEFClient _reefClient;
         private readonly JobRequestBuilder _jobRequestBuilder;
 
         private static readonly Logger Logger = 
Logger.GetLogger(typeof(HelloREEFYarn));
@@ -57,7 +58,7 @@ namespace Org.Apache.REEF.Examples.HelloREEF
         private readonly IList<string> _nodeNames;
 
         [Inject]
-        private HelloREEFYarn(IREEFClient reefClient, 
+        private HelloREEFYarn(IYarnREEFClient reefClient, 
             JobRequestBuilder jobRequestBuilder,
             [Parameter(typeof(NodeNames))] ISet<string> nodeNames)
         {
@@ -94,11 +95,26 @@ namespace Org.Apache.REEF.Examples.HelloREEF
                 .Build();
 
             var result = _reefClient.SubmitAndGetJobStatus(helloJobRequest);
+
+            LogApplicationReport();
             var state = PullFinalJobStatus(result);
             Logger.Log(Level.Info, "Application state : {0}.", state);
         }
 
         /// <summary>
+        /// Get application report and log.
+        /// </summary>
+        private void LogApplicationReport()
+        {
+            Logger.Log(Level.Info, "Getting Application report...");
+            var apps = _reefClient.GetApplicationReports().Result;
+            foreach (var r in apps)
+            {
+                Logger.Log(Level.Info, "Application report -- AppId {0}: 
{1}.", r.Key, r.Value.ToString());
+            }
+        }
+
+        /// <summary>
         /// This is to pull job final status until the Job is done
         /// </summary>
         /// <param name="jobSubmitionResult"></param>

Reply via email to