Repository: reef Updated Branches: refs/heads/master 48d47fe0f -> ec7bc969f
[REEF-1175] Add support for network credential to O.A.R.Client.YarnClient This addressed the issue by: * Adding NetworkCredential support to HttpClient which delegates to .NEt * HTTPClient * Add interface IYarnRestClientCredential which allows user to specify * mechanism to access their credential source JIRA: [REEF-1175](https://issues.apache.org/jira/browse/REEF-1175) Pull Request: Closes #812 Project: http://git-wip-us.apache.org/repos/asf/reef/repo Commit: http://git-wip-us.apache.org/repos/asf/reef/commit/ec7bc969 Tree: http://git-wip-us.apache.org/repos/asf/reef/tree/ec7bc969 Diff: http://git-wip-us.apache.org/repos/asf/reef/diff/ec7bc969 Branch: refs/heads/master Commit: ec7bc969f3b3483f1a81a2b17762d4ae4dc6a3c4 Parents: 48d47fe Author: Anupam <[email protected]> Authored: Tue Jan 5 10:03:07 2016 -0800 Committer: Andrew Chung <[email protected]> Committed: Wed Feb 3 17:45:31 2016 -0800 ---------------------------------------------------------------------- .../HDInsightYarnClientTests.cs | 168 +++++++++++++++++++ .../Org.Apache.REEF.Client.Tests.csproj | 1 + .../Org.Apache.REEF.Client.csproj | 2 + .../YARN/RESTClient/HttpClient.cs | 4 +- .../YARN/RESTClient/IRestRequestExecutor.cs | 4 +- .../RESTClient/IYarnRestClientCredential.cs | 35 ++++ .../RESTClient/NullYarnRestClientCredential.cs | 38 +++++ .../YARN/YARNClientConfiguration.cs | 3 + 8 files changed, 251 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/reef/blob/ec7bc969/lang/cs/Org.Apache.REEF.Client.Tests/HDInsightYarnClientTests.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client.Tests/HDInsightYarnClientTests.cs b/lang/cs/Org.Apache.REEF.Client.Tests/HDInsightYarnClientTests.cs new file mode 100644 index 0000000..2f61756 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Client.Tests/HDInsightYarnClientTests.cs @@ -0,0 +1,168 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Threading.Tasks; +using Org.Apache.REEF.Client.Yarn.RestClient; +using Org.Apache.REEF.Client.YARN.RestClient; +using Org.Apache.REEF.Client.YARN.RestClient.DataModel; +using Org.Apache.REEF.Tang.Implementations.Tang; +using Org.Apache.REEF.Tang.Util; +using Xunit; + +namespace Org.Apache.REEF.Client.Tests +{ + public class HDInsightYarnClientTests + { + [Trait("Category", "Functional")] + [Fact(Skip = @"Requires HDInsight Cred")] + public async Task TestGetClusterInfo() + { + var injector = TangFactory.GetTang().NewInjector(); + var cred = new HDInsightTestCredential(); + var urlProvider = new HDInsightRMUrlProvider(); + injector.BindVolatileInstance(GenericType<IYarnRestClientCredential>.Class, cred); + injector.BindVolatileInstance(GenericType<IUrlProvider>.Class, urlProvider); + var client = injector.GetInstance<IYarnRMClient>(); + + var clusterInfo = await client.GetClusterInfoAsync(); + + Assert.NotNull(clusterInfo); + Assert.Equal(ClusterState.STARTED, clusterInfo.State); + Assert.True(clusterInfo.StartedOn > 0); + } + + [Trait("Category", "Functional")] + [Fact(Skip = @"Requires HDInsight Cred")] + public async Task TestGetClusterMetrics() + { + var injector = TangFactory.GetTang().NewInjector(); + var cred = new HDInsightTestCredential(); + var urlProvider = new HDInsightRMUrlProvider(); + injector.BindVolatileInstance(GenericType<IYarnRestClientCredential>.Class, cred); + injector.BindVolatileInstance(GenericType<IUrlProvider>.Class, urlProvider); + var client = injector.GetInstance<IYarnRMClient>(); + + var clusterMetrics = await client.GetClusterMetricsAsync(); + + Assert.NotNull(clusterMetrics); + Assert.True(clusterMetrics.TotalMB > 0); + Assert.True(clusterMetrics.ActiveNodes > 0); + } + + [Trait("Category", "Functional")] + [Fact(Skip = @"Requires HDInsight Cred")] + public async Task TestApplicationSubmissionAndQuery() + { + var injector = TangFactory.GetTang().NewInjector(); + var cred = new HDInsightTestCredential(); + var urlProvider = new HDInsightRMUrlProvider(); + injector.BindVolatileInstance(GenericType<IYarnRestClientCredential>.Class, cred); + injector.BindVolatileInstance(GenericType<IUrlProvider>.Class, urlProvider); + var client = injector.GetInstance<IYarnRMClient>(); + + var newApplication = await client.CreateNewApplicationAsync(); + + Assert.NotNull(newApplication); + Assert.False(string.IsNullOrEmpty(newApplication.ApplicationId)); + Assert.True(newApplication.MaximumResourceCapability.MemoryMB > 0); + Assert.True(newApplication.MaximumResourceCapability.VCores > 0); + + string applicationName = "REEFTEST_APPLICATION_" + Guid.NewGuid(); + Console.WriteLine(applicationName); + + const string anyApplicationType = "REEFTest"; + var submitApplicationRequest = new SubmitApplication + { + ApplicationId = newApplication.ApplicationId, + AmResource = new Resouce + { + MemoryMB = 500, + VCores = 1 + }, + ApplicationType = anyApplicationType, + ApplicationName = applicationName, + KeepContainersAcrossApplicationAttempts = false, + MaxAppAttempts = 1, + Priority = 1, + UnmanagedAM = false, + AmContainerSpec = new AmContainerSpec + { + Commands = new Commands + { + Command = @"DONTCARE" + }, + LocalResources = new LocalResources + { + Entries = new List<YARN.RestClient.DataModel.KeyValuePair<string, LocalResourcesValue>> + { + new YARN.RestClient.DataModel.KeyValuePair<string, LocalResourcesValue> + { + Key = "APPLICATIONWILLFAILBUTWEDONTCAREHERE", + Value = new LocalResourcesValue + { + Resource = "Foo", + Type = ResourceType.FILE, + Visibility = Visibility.APPLICATION + } + } + } + } + } + }; + + var application = await client.SubmitApplicationAsync(submitApplicationRequest); + + Assert.NotNull(application); + Assert.Equal(newApplication.ApplicationId, application.Id); + Assert.Equal(applicationName, application.Name); + Assert.Equal(anyApplicationType, application.ApplicationType); + + var getApplicationResult = client.GetApplicationAsync(newApplication.ApplicationId).GetAwaiter().GetResult(); + + Assert.NotNull(getApplicationResult); + Assert.Equal(newApplication.ApplicationId, getApplicationResult.Id); + Assert.Equal(applicationName, getApplicationResult.Name); + Assert.Equal(anyApplicationType, getApplicationResult.ApplicationType); + } + } + + public class HDInsightTestCredential : IYarnRestClientCredential + { + private const string UserName = @"foo"; + private const string Password = @"bar"; // TODO: Do not checkin!!! + private readonly ICredentials _credentials = new NetworkCredential(UserName, Password); + + public ICredentials Credentials + { + get { return _credentials; } + } + } + + public class HDInsightRMUrlProvider : IUrlProvider + { + private const string HDInsightUrl = "https://baz.azurehdinsight.net/";; + + public Task<IEnumerable<Uri>> GetUrlAsync() + { + return Task.FromResult(Enumerable.Repeat(new Uri(HDInsightUrl), 1)); + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/reef/blob/ec7bc969/lang/cs/Org.Apache.REEF.Client.Tests/Org.Apache.REEF.Client.Tests.csproj ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client.Tests/Org.Apache.REEF.Client.Tests.csproj b/lang/cs/Org.Apache.REEF.Client.Tests/Org.Apache.REEF.Client.Tests.csproj index 724fe05..6d72cdc 100644 --- a/lang/cs/Org.Apache.REEF.Client.Tests/Org.Apache.REEF.Client.Tests.csproj +++ b/lang/cs/Org.Apache.REEF.Client.Tests/Org.Apache.REEF.Client.Tests.csproj @@ -62,6 +62,7 @@ under the License. </Reference> </ItemGroup> <ItemGroup> + <Compile Include="HDInsightYarnClientTests.cs" /> <Compile Include="JobResourceUploaderTests.cs" /> <Compile Include="LegacyJobResourceUploaderTests.cs" /> <Compile Include="MultipleRMUrlProviderTests.cs" /> http://git-wip-us.apache.org/repos/asf/reef/blob/ec7bc969/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj b/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj index 91a2114..be6e3e2 100644 --- a/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj +++ b/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj @@ -103,7 +103,9 @@ under the License. <Compile Include="YARN\RestClient\IRequestFactory.cs" /> <Compile Include="YARN\RestClient\IRestClient.cs" /> <Compile Include="YARN\RestClient\ISerializer.cs" /> + <Compile Include="YARN\RestClient\IYarnRestClientCredential.cs" /> <Compile Include="YARN\RestClient\Method.cs" /> + <Compile Include="YARN\RestClient\NullYarnRestClientCredential.cs" /> <Compile Include="YARN\RestClient\RequestFactory.cs" /> <Compile Include="YARN\RestClient\RestClient.cs" /> <Compile Include="YARN\RestClient\RestJsonDeserializer.cs" /> http://git-wip-us.apache.org/repos/asf/reef/blob/ec7bc969/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/HttpClient.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/HttpClient.cs b/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/HttpClient.cs index 6d636d2..311b57b 100644 --- a/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/HttpClient.cs +++ b/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/HttpClient.cs @@ -30,10 +30,10 @@ namespace Org.Apache.REEF.Client.YARN.RestClient private readonly System.Net.Http.HttpClient _httpClient; [Inject] - private HttpClient() + private HttpClient(IYarnRestClientCredential yarnRestClientCredential) { _httpClient = new System.Net.Http.HttpClient( - new HttpClientRetryHandler(new WebRequestHandler()), + new HttpClientRetryHandler(new WebRequestHandler { Credentials = yarnRestClientCredential.Credentials }), disposeHandler: false); } http://git-wip-us.apache.org/repos/asf/reef/blob/ec7bc969/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/IRestRequestExecutor.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/IRestRequestExecutor.cs b/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/IRestRequestExecutor.cs index ca4e2bc..0091a57 100644 --- a/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/IRestRequestExecutor.cs +++ b/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/IRestRequestExecutor.cs @@ -5,9 +5,9 @@ // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, // software distributed under the License is distributed on an // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY http://git-wip-us.apache.org/repos/asf/reef/blob/ec7bc969/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/IYarnRestClientCredential.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/IYarnRestClientCredential.cs b/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/IYarnRestClientCredential.cs new file mode 100644 index 0000000..567188d --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/IYarnRestClientCredential.cs @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Net; +using Org.Apache.REEF.Tang.Annotations; + +namespace Org.Apache.REEF.Client.YARN.RestClient +{ + /// <summary> + /// Provides the credentials to be used by the REST client to + /// connect to YARN RM. + /// </summary> + [DefaultImplementation(typeof(NullYarnRestClientCredential))] + public interface IYarnRestClientCredential + { + /// <summary> + /// .NET credentials to be used by REST client + /// </summary> + ICredentials Credentials { get; } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/reef/blob/ec7bc969/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/NullYarnRestClientCredential.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/NullYarnRestClientCredential.cs b/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/NullYarnRestClientCredential.cs new file mode 100644 index 0000000..eecd3c6 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/NullYarnRestClientCredential.cs @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Net; +using Org.Apache.REEF.Tang.Annotations; + +namespace Org.Apache.REEF.Client.YARN.RestClient +{ + internal class NullYarnRestClientCredential : IYarnRestClientCredential + { + [Inject] + private NullYarnRestClientCredential() + { + } + + public ICredentials Credentials + { + get + { + return null; + } + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/reef/blob/ec7bc969/lang/cs/Org.Apache.REEF.Client/YARN/YARNClientConfiguration.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/YARNClientConfiguration.cs b/lang/cs/Org.Apache.REEF.Client/YARN/YARNClientConfiguration.cs index 0920cb7..04753fc 100644 --- a/lang/cs/Org.Apache.REEF.Client/YARN/YARNClientConfiguration.cs +++ b/lang/cs/Org.Apache.REEF.Client/YARN/YARNClientConfiguration.cs @@ -18,6 +18,7 @@ using Org.Apache.REEF.Client.API; using Org.Apache.REEF.Client.YARN; using Org.Apache.REEF.Client.YARN.Parameters; +using Org.Apache.REEF.Client.YARN.RestClient; using Org.Apache.REEF.Tang.Formats; using Org.Apache.REEF.Tang.Util; using Org.Apache.REEF.Utilities.Attributes; @@ -32,6 +33,7 @@ namespace Org.Apache.REEF.Client.Yarn public static readonly OptionalParameter<string> JobSubmissionFolderPrefix = new OptionalParameter<string>(); public static readonly OptionalParameter<string> SecurityTokenKind = new OptionalParameter<string>(); public static readonly OptionalParameter<string> SecurityTokenService = new OptionalParameter<string>(); + public static readonly OptionalImpl<IYarnRestClientCredential> YarnRestClientCredential = new OptionalImpl<IYarnRestClientCredential>(); public static ConfigurationModule ConfigurationModule = new YARNClientConfiguration() .BindImplementation(GenericType<IREEFClient>.Class, GenericType<YarnREEFClient>.Class) @@ -44,6 +46,7 @@ namespace Org.Apache.REEF.Client.Yarn " and ConfigurationModuleYARNRest would be merged.")] public static ConfigurationModule ConfigurationModuleYARNRest = new YARNClientConfiguration() .BindImplementation(GenericType<IREEFClient>.Class, GenericType<YarnREEFDotNetClient>.Class) + .BindImplementation(GenericType<IYarnRestClientCredential>.Class, YarnRestClientCredential) .BindNamedParameter(GenericType<JobSubmissionDirectoryPrefixParameter>.Class, JobSubmissionFolderPrefix) .BindNamedParameter(GenericType<SecurityTokenKindParameter>.Class, SecurityTokenKind) .BindNamedParameter(GenericType<SecurityTokenServiceParameter>.Class, SecurityTokenService)
