Repository: reef Updated Branches: refs/heads/master 9fb6cc41a -> 85bae60ab
[REEF-1878] Allow multiple tokens to pass in from client * Added SecurityTokenWriter at .Net side * Added SecurityTOkenWriter at Java side * Added named parameters to allow .Net clients to pass tokens * Modified Java side to allow multiple tokens to be pass in * Support backward compitability * Added tests. JIRA: [REEF-1878](https://issues.apache.org/jira/browse/REEF-1878) This closes #1372 Project: http://git-wip-us.apache.org/repos/asf/reef/repo Commit: http://git-wip-us.apache.org/repos/asf/reef/commit/85bae60a Tree: http://git-wip-us.apache.org/repos/asf/reef/tree/85bae60a Diff: http://git-wip-us.apache.org/repos/asf/reef/diff/85bae60a Branch: refs/heads/master Commit: 85bae60ab0b988b143ddd49f8f4295475ebdf49c Parents: 9fb6cc4 Author: Julia Wang <[email protected]> Authored: Fri Sep 1 19:05:53 2017 -0700 Committer: Sergiy Matusevych <[email protected]> Committed: Mon Sep 25 12:14:48 2017 -0700 ---------------------------------------------------------------------- lang/common/token/avro/SecurityToken.avsc | 52 ++++ .../Avro/YARN/SecurityToken.cs | 87 +++++++ .../Org.Apache.REEF.Client.csproj | 3 + .../YARN/Parameters/SecurityTokenParameters.cs | 8 +- .../YARN/Parameters/SecurityTokenStrings.cs | 30 +++ .../YARN/SecurityTokenWriter.cs | 81 +++++++ .../YARN/YARNClientConfiguration.cs | 3 + .../YARN/YARNREEFClient.cs | 2 + .../YARN/YarnREEFParamSerializer.cs | 53 +++- .../Files/REEFFileNames.cs | 15 ++ .../HelloREEFYarn.cs | 24 +- .../Org.Apache.REEF.Examples.HelloREEF.csproj | 4 + .../packages.config | 1 + .../Functional/Bridge/TestSecurityToken.cs | 240 +++++++++++++++++++ .../Org.Apache.REEF.Tests.csproj | 9 +- lang/java/reef-bridge-client/pom.xml | 12 + .../bridge/client/SecurityTokensReader.java | 82 +++++++ .../bridge/client/YarnJobSubmissionClient.java | 106 +++++--- .../runtime/common/files/REEFFileNames.java | 29 ++- 19 files changed, 793 insertions(+), 48 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/reef/blob/85bae60a/lang/common/token/avro/SecurityToken.avsc ---------------------------------------------------------------------- diff --git a/lang/common/token/avro/SecurityToken.avsc b/lang/common/token/avro/SecurityToken.avsc new file mode 100644 index 0000000..5072ff1 --- /dev/null +++ b/lang/common/token/avro/SecurityToken.avsc @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +[ + /** + * Avro schema for serialized security token + */ + { + "namespace": "org.apache.reef.bridge.client", + + "name": "SecurityToken", + "doc": "Security token", + "type": "record", + "fields": [ + { + "name": "kind", + "doc": "Kind of the security token", + "type": "string" + }, + { + "name": "service", + "doc": "Token service name", + "type": "string" + }, + { + "name": "key", + "type": "bytes", + "doc": "Token key" + }, + { + "name": "password", + "type": "bytes", + "doc": "Token password" + } + ] + } +] http://git-wip-us.apache.org/repos/asf/reef/blob/85bae60a/lang/cs/Org.Apache.REEF.Client/Avro/YARN/SecurityToken.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/Avro/YARN/SecurityToken.cs b/lang/cs/Org.Apache.REEF.Client/Avro/YARN/SecurityToken.cs new file mode 100644 index 0000000..199a361 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Client/Avro/YARN/SecurityToken.cs @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Org.Apache.REEF.Client.Avro.YARN +{ + using System.Runtime.Serialization; + + /// <summary> + /// Used to serialize and deserialize Avro record org.apache.reef.bridge.client.SecurityToken. + /// </summary> + [DataContract(Namespace = "org.apache.reef.bridge.client")] + public partial class SecurityToken + { + private const string JsonSchema = @"{""type"":""record"",""name"":""org.apache.reef.bridge.client.SecurityToken"",""doc"":""Security token"",""fields"":[{""name"":""kind"",""doc"":""Kind of the security token"",""type"":""string""},{""name"":""service"",""doc"":""Token service name"",""type"":""string""},{""name"":""key"",""doc"":""Token key"",""type"":""bytes""},{""name"":""password"",""doc"":""Token password"",""type"":""bytes""}]}"; + + /// <summary> + /// Gets the schema. + /// </summary> + public static string Schema + { + get + { + return JsonSchema; + } + } + + /// <summary> + /// Gets or sets the kind field. + /// </summary> + [DataMember] + public string kind { get; set; } + + /// <summary> + /// Gets or sets the service field. + /// </summary> + [DataMember] + public string service { get; set; } + + /// <summary> + /// Gets or sets the key field. + /// </summary> + [DataMember] + public byte[] key { get; set; } + + /// <summary> + /// Gets or sets the password field. + /// </summary> + [DataMember] + public byte[] password { get; set; } + + /// <summary> + /// Initializes a new instance of the <see cref="SecurityToken"/> class. + /// </summary> + public SecurityToken() + { + } + + /// <summary> + /// Initializes a new instance of the <see cref="SecurityToken"/> class. + /// </summary> + /// <param name="kind">The kind.</param> + /// <param name="service">The service.</param> + /// <param name="key">The key.</param> + /// <param name="password">The password.</param> + public SecurityToken(string kind, string service, byte[] key, byte[] password) + { + this.kind = kind; + this.service = service; + this.key = key; + this.password = password; + } + } +} http://git-wip-us.apache.org/repos/asf/reef/blob/85bae60a/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj b/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj index 791af4f..7748895 100644 --- a/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj +++ b/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj @@ -119,6 +119,7 @@ under the License. <Compile Include="YARN\Parameters\DriverStderrFilePath.cs" /> <Compile Include="YARN\Parameters\DriverStdoutFilePath.cs" /> <Compile Include="YARN\Parameters\FileSystemUrl.cs" /> + <Compile Include="YARN\Parameters\SecurityTokenStrings.cs" /> <Compile Include="YARN\RestClient\DataModel\KillApplication.cs" /> <Compile Include="YARN\RestClient\HttpClient.cs" /> <Compile Include="YARN\RestClient\IDeserializer.cs" /> @@ -136,6 +137,8 @@ under the License. <Compile Include="YARN\RestClient\RestRequest.cs" /> <Compile Include="YARN\RestClient\RestResponse.cs" /> <Compile Include="YARN\RestClient\HttpClientRetryHandler.cs" /> + <Compile Include="Avro\YARN\SecurityToken.cs" /> + <Compile Include="YARN\SecurityTokenWriter.cs" /> <Compile Include="YARN\WindowsYarnJobCommandProvider.cs" /> <Compile Include="YARN\JobResource.cs" /> <Compile Include="YARN\JobSubmissionDirectoryProvider.cs" /> http://git-wip-us.apache.org/repos/asf/reef/blob/85bae60a/lang/cs/Org.Apache.REEF.Client/YARN/Parameters/SecurityTokenParameters.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/Parameters/SecurityTokenParameters.cs b/lang/cs/Org.Apache.REEF.Client/YARN/Parameters/SecurityTokenParameters.cs index a6a8092..88845a8 100644 --- a/lang/cs/Org.Apache.REEF.Client/YARN/Parameters/SecurityTokenParameters.cs +++ b/lang/cs/Org.Apache.REEF.Client/YARN/Parameters/SecurityTokenParameters.cs @@ -19,13 +19,17 @@ using Org.Apache.REEF.Tang.Annotations; namespace Org.Apache.REEF.Client.YARN.Parameters { - [NamedParameter("Security token kind.", defaultValue: "NULL")] + [System.Obsolete("TODO[JIRA REEF-1887] Deprecated. Remove in REEF 0.18.")] + [NamedParameter("Security token kind.", defaultValue: DefaultTokenKind)] public sealed class SecurityTokenKindParameter : Name<string> { + public const string DefaultTokenKind = "NULL"; } - [NamedParameter("Security token service name.", defaultValue: "NULL")] + [System.Obsolete("TODO[JIRA REEF-1887] Deprecated. Remove in REEF 0.18.")] + [NamedParameter("Security token service name.", defaultValue: DefaultService)] public sealed class SecurityTokenServiceParameter : Name<string> { + public const string DefaultService = "NULL"; } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/reef/blob/85bae60a/lang/cs/Org.Apache.REEF.Client/YARN/Parameters/SecurityTokenStrings.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/Parameters/SecurityTokenStrings.cs b/lang/cs/Org.Apache.REEF.Client/YARN/Parameters/SecurityTokenStrings.cs new file mode 100644 index 0000000..88ad649 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Client/YARN/Parameters/SecurityTokenStrings.cs @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Collections.Generic; +using Org.Apache.REEF.Tang.Annotations; + +namespace Org.Apache.REEF.Client.YARN.Parameters +{ + /// <summary> + /// Named parameter that contains a set of serialized tokens. + /// </summary> + [NamedParameter("Serialized SurityToken Info", "SecurityTokenStrings")] + public class SecurityTokenStrings : Name<ISet<string>> + { + } +} http://git-wip-us.apache.org/repos/asf/reef/blob/85bae60a/lang/cs/Org.Apache.REEF.Client/YARN/SecurityTokenWriter.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/SecurityTokenWriter.cs b/lang/cs/Org.Apache.REEF.Client/YARN/SecurityTokenWriter.cs new file mode 100644 index 0000000..060064c --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Client/YARN/SecurityTokenWriter.cs @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Newtonsoft.Json; +using Org.Apache.REEF.Client.YARN.Parameters; +using Org.Apache.REEF.Tang.Annotations; +using Org.Apache.REEF.Utilities.Logging; +using Microsoft.Hadoop.Avro; +using Org.Apache.REEF.Client.Avro.YARN; +using Org.Apache.REEF.Common.Files; + +namespace Org.Apache.REEF.Client.YARN +{ + /// <summary> + /// Serialize a set of security tokens into a file. + /// </summary> + internal class SecurityTokenWriter + { + private static readonly Logger Logger = Logger.GetLogger(typeof(SecurityTokenWriter)); + + private readonly IAvroSerializer<SecurityToken> _avroSerializer = AvroSerializer.Create<SecurityToken>(); + private readonly List<SecurityToken> _tokens; + + private readonly string _securityTokensFile; + + /// <summary> + /// Injectable constructor that accepts a set of serialized tokens. + /// Each serialized token string in the set is a serialized SecurityToken by JsonConvert + /// </summary> + /// <param name="serializedTokenStrings">Serialized token strings</param> + /// <param name="reefFileNames">REEF file name constants</param> + [Inject] + private SecurityTokenWriter(REEFFileNames reefFileNames, + [Parameter(typeof(SecurityTokenStrings))] ISet<string> serializedTokenStrings) + { + _securityTokensFile = reefFileNames.GetSecurityTokenFileName(); + _tokens = serializedTokenStrings.Select(serializedToken => + { + var token = JsonConvert.DeserializeObject<SecurityToken>(serializedToken); + return new SecurityToken(token.kind, token.service, token.key, token.password); + }).ToList(); + } + + /// <summary> + /// Write SecurityToken objects to SecurityTokenFile using IAvroSerializer. + /// </summary> + public void WriteTokensToFile() + { + Logger.Log(Level.Verbose, "Write {0} tokens to file: {1}.", _tokens.Count, _securityTokensFile); + + if (_tokens.Count > 0) + { + using (var stream = File.OpenWrite(_securityTokensFile)) + { + foreach (var token in _tokens) + { + Logger.Log(Level.Verbose, "Write token {0} to file.", token.kind); + _avroSerializer.Serialize(stream, token); + } + } + } + } + } +} http://git-wip-us.apache.org/repos/asf/reef/blob/85bae60a/lang/cs/Org.Apache.REEF.Client/YARN/YARNClientConfiguration.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/YARNClientConfiguration.cs b/lang/cs/Org.Apache.REEF.Client/YARN/YARNClientConfiguration.cs index 69d715e..34e48da 100644 --- a/lang/cs/Org.Apache.REEF.Client/YARN/YARNClientConfiguration.cs +++ b/lang/cs/Org.Apache.REEF.Client/YARN/YARNClientConfiguration.cs @@ -34,6 +34,7 @@ namespace Org.Apache.REEF.Client.Yarn public static readonly OptionalParameter<string> SecurityTokenKind = new OptionalParameter<string>(); public static readonly OptionalParameter<string> SecurityTokenService = new OptionalParameter<string>(); public static readonly OptionalImpl<IYarnRestClientCredential> YarnRestClientCredential = new OptionalImpl<IYarnRestClientCredential>(); + public static readonly OptionalParameter<string> SecurityTokenStr = new OptionalParameter<string>(); /// <summary> /// URL for store. For Hadoop file system, it is set in fs.defaultFS as default by YARN environment. Client doesn't need to @@ -49,6 +50,7 @@ namespace Org.Apache.REEF.Client.Yarn .BindNamedParameter(GenericType<JobSubmissionDirectoryPrefixParameter>.Class, JobSubmissionFolderPrefix) .BindNamedParameter(GenericType<SecurityTokenKindParameter>.Class, SecurityTokenKind) .BindNamedParameter(GenericType<SecurityTokenServiceParameter>.Class, SecurityTokenService) + .BindSetEntry(GenericType<SecurityTokenStrings>.Class, SecurityTokenStr) .BindNamedParameter(GenericType<FileSystemUrl>.Class, FileSystemUrl) .Build(); @@ -61,6 +63,7 @@ namespace Org.Apache.REEF.Client.Yarn .BindNamedParameter(GenericType<SecurityTokenKindParameter>.Class, SecurityTokenKind) .BindNamedParameter(GenericType<SecurityTokenServiceParameter>.Class, SecurityTokenService) .BindNamedParameter(GenericType<FileSystemUrl>.Class, FileSystemUrl) + .BindSetEntry(GenericType<SecurityTokenStrings>.Class, SecurityTokenStr) .Build(); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/reef/blob/85bae60a/lang/cs/Org.Apache.REEF.Client/YARN/YARNREEFClient.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/YARNREEFClient.cs b/lang/cs/Org.Apache.REEF.Client/YARN/YARNREEFClient.cs index 4559c73..b4e5ad7 100644 --- a/lang/cs/Org.Apache.REEF.Client/YARN/YARNREEFClient.cs +++ b/lang/cs/Org.Apache.REEF.Client/YARN/YARNREEFClient.cs @@ -158,6 +158,8 @@ namespace Org.Apache.REEF.Client.Yarn { _driverFolderPreparationHelper.PrepareDriverFolder(jobRequest.AppParameters, driverFolderPath); + _paramSerializer.WriteSecurityTokens(); + // TODO: Remove this when we have a generalized way to pass config to java var paramInjector = TangFactory.GetTang().NewInjector(jobRequest.DriverConfigurations.ToArray()); var submissionJobArgsFilePath = _paramSerializer.SerializeJobFile(jobRequest.JobParameters, paramInjector, driverFolderPath); http://git-wip-us.apache.org/repos/asf/reef/blob/85bae60a/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFParamSerializer.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFParamSerializer.cs b/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFParamSerializer.cs index bcaf6fd..c09eaad 100644 --- a/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFParamSerializer.cs +++ b/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFParamSerializer.cs @@ -19,13 +19,13 @@ using System.IO; using Org.Apache.REEF.Client.API; using Org.Apache.REEF.Client.Avro; using Org.Apache.REEF.Client.Avro.YARN; -using Org.Apache.REEF.Client.Yarn; using Org.Apache.REEF.Client.YARN.Parameters; using Org.Apache.REEF.Common.Avro; using Org.Apache.REEF.Common.Files; using Org.Apache.REEF.Driver.Bridge; using Org.Apache.REEF.Tang.Annotations; using Org.Apache.REEF.Tang.Interface; +using Org.Apache.REEF.Utilities.Logging; using Org.Apache.REEF.Wake.Remote.Parameters; namespace Org.Apache.REEF.Client.YARN @@ -35,24 +35,61 @@ namespace Org.Apache.REEF.Client.YARN /// </summary> internal sealed class YarnREEFParamSerializer { + private static readonly Logger Logger = Logger.GetLogger(typeof(YarnREEFParamSerializer)); + private readonly REEFFileNames _fileNames; + + /// <summary> + /// Security token kind name. Used for single token case. + /// </summary> + [System.Obsolete("TODO[JIRA REEF-1887] Deprecated. Remove in REEF 0.18.")] private readonly string _securityTokenKind; + + /// <summary> + /// Security token service name. Used for single token case. + /// </summary> + [System.Obsolete("TODO[JIRA REEF-1887] Deprecated. Remove in REEF 0.18.")] private readonly string _securityTokenService; + + /// <summary> + /// File system URL + /// </summary> private readonly string _fileSystemUrl; + + /// <summary> + /// Job submission folder relative path + /// </summary> private readonly string _jobSubmissionPrefix; + /// <summary> + /// Security token writer that parses and writes token information. + /// It can process multiple tokens. + /// </summary> + private readonly SecurityTokenWriter _securityTokenWriter; + + /// <summary> + /// Serialize parameters and tokens for Java client. + /// </summary> + /// <param name="fileNames">REEF file name class which contains file names.</param> + /// <param name="securityTokenWriter">SecurityTokenWriter which writes security token info.</param> + /// <param name="securityTokenKind">Security token kind name.</param> + /// <param name="securityTokenService">Security token service name.</param> + /// <param name="fileSystemUrl">File system URL.</param> + /// <param name="jobSubmissionPrefix">Job submission folder. e.g: fileSystemUrl/jobSubmissionPrefix/</param> [Inject] private YarnREEFParamSerializer( REEFFileNames fileNames, + SecurityTokenWriter securityTokenWriter, [Parameter(typeof(SecurityTokenKindParameter))] string securityTokenKind, [Parameter(typeof(SecurityTokenServiceParameter))] string securityTokenService, [Parameter(typeof(FileSystemUrl))] string fileSystemUrl, [Parameter(typeof(JobSubmissionDirectoryPrefixParameter))] string jobSubmissionPrefix) { _fileNames = fileNames; + _securityTokenWriter = securityTokenWriter; _jobSubmissionPrefix = jobSubmissionPrefix; - _securityTokenKind = securityTokenKind; _fileSystemUrl = fileSystemUrl; + _securityTokenKind = securityTokenKind; _securityTokenService = securityTokenService; } @@ -123,9 +160,11 @@ namespace Org.Apache.REEF.Client.YARN var avroYarnClusterJobSubmissionParameters = new AvroYarnClusterJobSubmissionParameters { - yarnJobSubmissionParameters = avroYarnJobSubmissionParameters, + // TODO[JIRA REEF-1887] Deprecated. Remove in REEF 0.18. securityTokenKind = _securityTokenKind, securityTokenService = _securityTokenService, + + yarnJobSubmissionParameters = avroYarnJobSubmissionParameters, driverMemory = jobParameters.DriverMemoryInMB, maxApplicationSubmissions = jobParameters.MaxApplicationSubmissions, driverStdoutFilePath = string.IsNullOrWhiteSpace(jobParameters.StdoutFilePath.Value) ? @@ -136,5 +175,13 @@ namespace Org.Apache.REEF.Client.YARN return AvroJsonSerializer<AvroYarnClusterJobSubmissionParameters>.ToBytes(avroYarnClusterJobSubmissionParameters); } + + /// <summary> + /// Write Token info. + /// </summary> + internal void WriteSecurityTokens() + { + _securityTokenWriter.WriteTokensToFile(); + } } } http://git-wip-us.apache.org/repos/asf/reef/blob/85bae60a/lang/cs/Org.Apache.REEF.Common/Files/REEFFileNames.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Common/Files/REEFFileNames.cs b/lang/cs/Org.Apache.REEF.Common/Files/REEFFileNames.cs index dd5be54..5cb272e 100644 --- a/lang/cs/Org.Apache.REEF.Common/Files/REEFFileNames.cs +++ b/lang/cs/Org.Apache.REEF.Common/Files/REEFFileNames.cs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +using System; using System.Diagnostics.CodeAnalysis; using System.IO; using Org.Apache.REEF.Tang.Annotations; @@ -53,6 +54,7 @@ namespace Org.Apache.REEF.Common.Files private const string BRIDGE_EXE_CONFIG_NAME = "Org.Apache.REEF.Bridge.exe.config"; private const string SECURITY_TOKEN_IDENTIFIER_FILE = "SecurityTokenId"; private const string SECURITY_TOKEN_PASSWORD_FILE = "SecurityTokenPwd"; + private const string SECURITY_TOKEN_FILE = "SecurityTokens.json"; private const string APP_SUBMISSION_PARAMETERS_FILE = "app-submission-params.json"; private const string JOB_SUBMISSION_PARAMETERS_FILE = "job-submission-params.json"; private const string YARN_DEFAULT_DRIVER_OUT_VAR = "<LOG_DIR>"; @@ -297,6 +299,7 @@ namespace Org.Apache.REEF.Common.Files /// The filename for security token identifier /// </summary> /// <returns>filename which contains raw bytes of security token identifier</returns> + [Obsolete("TODO[JIRA REEF-1887] Use GetSecurityTokenFileName() for consolidated token id and password information. Remove in REEF 0.18.")] [Unstable("0.13", "Security token should be handled by .NET only REEF client in the future")] public string GetSecurityTokenIdentifierFileName() { @@ -307,6 +310,7 @@ namespace Org.Apache.REEF.Common.Files /// The filename for security token password /// </summary> /// <returns>filename which contains raw bytes of security token password</returns> + [Obsolete("TODO[JIRA REEF-1887] Use GetSecurityTokenFileName() for consolidated token id and password information. Remove in REEF 0.18.")] [Unstable("0.13", "Security token should be handled by .NET only REEF client in the future")] public string GetSecurityTokenPasswordFileName() { @@ -314,6 +318,17 @@ namespace Org.Apache.REEF.Common.Files } /// <summary> + /// File name for security token information. + /// TODO[JIRA REEF-1887] It supersedes GetSecurityTokenPasswordFileName() and GetSecurityTokenIdentifierFileName(). + /// Remove this comment line when REEF-1887 is done. + /// </summary> + /// <returns>Returns security token file name.</returns> + public string GetSecurityTokenFileName() + { + return SECURITY_TOKEN_FILE; + } + + /// <summary> /// The file name of the PID file created in the current working directory of the process. /// This is similar to the file name in the PIDStoreHandler.java /// </summary> http://git-wip-us.apache.org/repos/asf/reef/blob/85bae60a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEFYarn.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEFYarn.cs b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEFYarn.cs index b7564f6..917e3eb 100644 --- a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEFYarn.cs +++ b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEFYarn.cs @@ -16,10 +16,12 @@ // under the License. using System.Collections.Generic; -using System.IO; using System.Linq; +using System.Text; using System.Threading; +using Newtonsoft.Json; using Org.Apache.REEF.Client.API; +using Org.Apache.REEF.Client.Avro.YARN; using Org.Apache.REEF.Client.Common; using Org.Apache.REEF.Client.Yarn; using Org.Apache.REEF.Client.YARN; @@ -30,6 +32,7 @@ using Org.Apache.REEF.Tang.Implementations.Configuration; using Org.Apache.REEF.Tang.Implementations.Tang; using Org.Apache.REEF.Tang.Interface; using Org.Apache.REEF.Tang.Util; +using Org.Apache.REEF.Utilities; using Org.Apache.REEF.Utilities.Logging; namespace Org.Apache.REEF.Examples.HelloREEF @@ -155,23 +158,24 @@ namespace Org.Apache.REEF.Examples.HelloREEF /// <summary> /// Get runtime configuration /// </summary> - /// <returns></returns> private static IConfiguration GetRuntimeConfiguration(string[] args) { - var c = YARNClientConfiguration.ConfigurationModule - .Set(YARNClientConfiguration.SecurityTokenKind, TrustedApplicationTokenIdentifier) - .Set(YARNClientConfiguration.SecurityTokenService, TrustedApplicationTokenIdentifier) + var token = new SecurityToken( + TrustedApplicationTokenIdentifier, + TrustedApplicationTokenIdentifier, + ByteUtilities.StringToByteArrays(args[0]), + Encoding.ASCII.GetBytes(args[1])); + + var clientConfig = YARNClientConfiguration.ConfigurationModule + .Set(YARNClientConfiguration.SecurityTokenStr, JsonConvert.SerializeObject(token)) .Build(); - File.WriteAllText(SecurityTokenId, args[0]); - File.WriteAllText(SecurityTokenPwd, args[1]); - - IConfiguration tcpPortConfig = TcpPortConfigurationModule.ConfigurationModule + var tcpPortConfig = TcpPortConfigurationModule.ConfigurationModule .Set(TcpPortConfigurationModule.PortRangeStart, args.Length > 2 ? args[2] : DefaultPortRangeStart) .Set(TcpPortConfigurationModule.PortRangeCount, args.Length > 3 ? args[3] : DefaultPortRangeCount) .Build(); - return Configurations.Merge(c, tcpPortConfig); + return Configurations.Merge(clientConfig, tcpPortConfig); } /// <summary> http://git-wip-us.apache.org/repos/asf/reef/blob/85bae60a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/Org.Apache.REEF.Examples.HelloREEF.csproj ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/Org.Apache.REEF.Examples.HelloREEF.csproj b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/Org.Apache.REEF.Examples.HelloREEF.csproj index be7fed1..2cce0ee 100644 --- a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/Org.Apache.REEF.Examples.HelloREEF.csproj +++ b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/Org.Apache.REEF.Examples.HelloREEF.csproj @@ -18,6 +18,10 @@ <UseVSHostingProcess>false</UseVSHostingProcess> </PropertyGroup> <ItemGroup> + <Reference Include="Newtonsoft.Json, Version=10.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL"> + <HintPath>..\packages\Newtonsoft.Json.10.0.3\lib\net45\Newtonsoft.Json.dll</HintPath> + <Private>True</Private> + </Reference> <Reference Include="System" /> <Reference Include="System.Core" /> <Reference Include="System.Xml.Linq" /> http://git-wip-us.apache.org/repos/asf/reef/blob/85bae60a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/packages.config ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/packages.config b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/packages.config index d952982..13ca20f 100644 --- a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/packages.config +++ b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/packages.config @@ -19,4 +19,5 @@ under the License. --> <packages> <package id="StyleCop.MSBuild" version="4.7.49.1" targetFramework="net45" developmentDependency="true" /> + <package id="Newtonsoft.Json" version="10.0.3" targetFramework="net451" /> </packages> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/reef/blob/85bae60a/lang/cs/Org.Apache.REEF.Tests/Functional/Bridge/TestSecurityToken.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Tests/Functional/Bridge/TestSecurityToken.cs b/lang/cs/Org.Apache.REEF.Tests/Functional/Bridge/TestSecurityToken.cs new file mode 100644 index 0000000..8e833fe --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Tests/Functional/Bridge/TestSecurityToken.cs @@ -0,0 +1,240 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Threading; +using Newtonsoft.Json; +using Org.Apache.REEF.Client.API; +using Org.Apache.REEF.Client.Avro.YARN; +using Org.Apache.REEF.Client.Common; +using Org.Apache.REEF.Client.Yarn; +using Org.Apache.REEF.Client.YARN.RestClient.DataModel; +using Org.Apache.REEF.Common.Files; +using Org.Apache.REEF.Driver; +using Org.Apache.REEF.Examples.HelloREEF; +using Org.Apache.REEF.Tang.Implementations.Configuration; +using Org.Apache.REEF.Tang.Implementations.Tang; +using Org.Apache.REEF.Tang.Interface; +using Org.Apache.REEF.Tang.Util; +using Org.Apache.REEF.Utilities; +using Org.Apache.REEF.Utilities.Logging; +using Xunit; + +namespace Org.Apache.REEF.Tests.Functional.Bridge +{ + /// <summary> + /// Test set security tokens + /// </summary> + [Collection("FunctionalTests")] + public class TestSecurityToken + { + private static readonly Logger Logger = Logger.GetLogger(typeof(TestSecurityToken)); + + private const string DefaultPortRangeStart = "2000"; + private const string DefaultPortRangeCount = "20"; + + private const string Identifier1 = "TrustedApplicationTokenIdentifier"; + private const string Identifier2 = "TrustedApplicationTokenIdentifier2"; + private const string TokenKey1 = "TrustedApplication001"; + private const string TokenKey2 = "TrustedApplication002"; + private const string Password1 = "none"; + private const string Password2 = "none"; + + /// <summary> + /// This is to test pass multiple tokens from client for Yarn application + /// </summary> + [Fact] + [Trait("Environment", "Yarn")] + [Trait("Priority", "1")] + [Trait("Description", "Run CLR Test on Yarn")] + public void TestSetMultipleSecurityToken() + { + TestRun(GetRuntimeConfigurationForMultipleTokens()); + } + + /// <summary> + /// This is to test write token in old approach for backward compatibility checking + /// </summary> + [Fact] + [Trait("Environment", "Yarn")] + [Trait("Priority", "1")] + [Trait("Description", "Run CLR Test on Yarn")] + [Obsolete("TODO[JIRA REEF-1887] Remove in REEF 0.18.")] + public void TestSecurityTokenBackwardCompatibility() + { + TestRun(GetRuntimeConfigurationBackwardComp()); + } + + /// <summary> + /// This is to test passing one token from client for Yarn application + /// </summary> + [Fact] + [Trait("Environment", "Yarn")] + [Trait("Priority", "1")] + [Trait("Description", "Run CLR Test on Yarn")] + public void TestSetOneSecurityToken() + { + TestRun(GetRuntimeConfigurationForSingleToken()); + } + + /// <summary> + /// Test run for the runtime in the given injector. + /// </summary> + /// <param name="config">runtime configuration.</param> + private void TestRun(IConfiguration config) + { + IInjector injector = TangFactory.GetTang().NewInjector(config); + + var reefClient = injector.GetInstance<IREEFClient>(); + var jobRequestBuilder = injector.GetInstance<JobRequestBuilder>(); + + var jobSubmission = jobRequestBuilder + .AddDriverConfiguration(GetDriverConfig()) + .AddGlobalAssemblyForType(typeof(HelloDriver)) + .SetJobIdentifier("MyTestJob") + .SetJavaLogLevel(JavaLoggingSetting.Verbose) + .Build(); + + var result = reefClient.SubmitAndGetJobStatus(jobSubmission); + var state = PullFinalJobStatus(result); + Logger.Log(Level.Info, "Application final state : {0}.", state); + Assert.Equal(FinalState.SUCCEEDED, state); + } + + /// <summary> + /// Use HelloDriver in the test + /// </summary> + /// <returns>Return driver configuration.</returns> + private IConfiguration GetDriverConfig() + { + return DriverConfiguration.ConfigurationModule + .Set(DriverConfiguration.OnEvaluatorAllocated, GenericType<HelloDriver>.Class) + .Set(DriverConfiguration.OnDriverStarted, GenericType<HelloDriver>.Class) + .Build(); + } + + /// <summary> + /// Pull job final status until the Job is done + /// </summary> + /// <param name="jobSubmitionResult"></param> + /// <returns>Return final state of the job.</returns> + private FinalState PullFinalJobStatus(IJobSubmissionResult jobSubmitionResult) + { + int n = 0; + var state = jobSubmitionResult.FinalState; + while (state.Equals(FinalState.UNDEFINED) && n++ < 100) + { + Thread.Sleep(3000); + state = jobSubmitionResult.FinalState; + } + return state; + } + + /// <summary> + /// Get runtime configuration. + /// Bind tokens to YarnClientCnfiguration. + /// </summary> + /// <returns>Return runtime configuration for multiple tokens.</returns> + private static IConfiguration GetRuntimeConfigurationForMultipleTokens() + { + var yarnClientConfigModule = YARNClientConfiguration.ConfigurationModule; + foreach (var t in CreateTestTokens()) + { + yarnClientConfigModule = yarnClientConfigModule.Set(YARNClientConfiguration.SecurityTokenStr, t); + } + + return Configurations.Merge(yarnClientConfigModule.Build(), TcpPortConfig()); + } + + /// <summary> + /// Get runtime configuration. + /// Bind token to YarnClientCnfiguration. + /// </summary> + /// <returns>Return runtime configuration for single token.</returns> + private static IConfiguration GetRuntimeConfigurationForSingleToken() + { + var yarnClientConfigModule = YARNClientConfiguration.ConfigurationModule + .Set(YARNClientConfiguration.SecurityTokenStr, CreateTestToken()); + + return Configurations.Merge(yarnClientConfigModule.Build(), TcpPortConfig()); + } + + /// <summary> + /// Get runtime configuration and token with old approach + /// </summary> + /// <returns>Return runtime configuration for old approach.</returns> + [Obsolete("TODO[JIRA REEF-1887] Remove in REEF 0.18.")] + private static IConfiguration GetRuntimeConfigurationBackwardComp() + { + var reefFileNames = TangFactory.GetTang().NewInjector().GetInstance<REEFFileNames>(); + + var c = YARNClientConfiguration.ConfigurationModule + .Set(YARNClientConfiguration.SecurityTokenKind, Identifier1) + .Set(YARNClientConfiguration.SecurityTokenService, Identifier1) + .Build(); + + File.WriteAllText(reefFileNames.GetSecurityTokenIdentifierFileName(), TokenKey1); + File.WriteAllText(reefFileNames.GetSecurityTokenPasswordFileName(), Password1); + + return Configurations.Merge(c, TcpPortConfig()); + } + + private static IConfiguration TcpPortConfig() + { + var tcpPortConfig = TcpPortConfigurationModule.ConfigurationModule + .Set(TcpPortConfigurationModule.PortRangeStart, DefaultPortRangeStart) + .Set(TcpPortConfigurationModule.PortRangeCount, DefaultPortRangeCount) + .Build(); + return tcpPortConfig; + } + + internal static IList<string> CreateTestTokens() + { + var t1 = new SecurityToken( + Identifier1, + Identifier1, + ByteUtilities.StringToByteArrays(TokenKey1), + Encoding.ASCII.GetBytes(Password1)); + + var t2 = new SecurityToken( + Identifier2, + Identifier2, + ByteUtilities.StringToByteArrays(TokenKey2), + Encoding.ASCII.GetBytes(Password2)); + + return new List<string>() + { + JsonConvert.SerializeObject(t1), + JsonConvert.SerializeObject(t2) + }; + } + + internal static string CreateTestToken() + { + var t1 = new SecurityToken( + Identifier1, + Identifier1, + ByteUtilities.StringToByteArrays(TokenKey1), + Encoding.ASCII.GetBytes(Password1)); + + return JsonConvert.SerializeObject(t1); + } + } +} http://git-wip-us.apache.org/repos/asf/reef/blob/85bae60a/lang/cs/Org.Apache.REEF.Tests/Org.Apache.REEF.Tests.csproj ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Tests/Org.Apache.REEF.Tests.csproj b/lang/cs/Org.Apache.REEF.Tests/Org.Apache.REEF.Tests.csproj index a4301b7..252bf75 100644 --- a/lang/cs/Org.Apache.REEF.Tests/Org.Apache.REEF.Tests.csproj +++ b/lang/cs/Org.Apache.REEF.Tests/Org.Apache.REEF.Tests.csproj @@ -1,4 +1,4 @@ -<?xml version="1.0" encoding="utf-8"?> +<?xml version="1.0" encoding="utf-8"?> <!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file @@ -83,6 +83,7 @@ under the License. <Compile Include="Functional\Bridge\TestContextStack.cs" /> <Compile Include="Functional\Bridge\TestDriverConcurrency.cs" /> <Compile Include="Functional\Bridge\TestFailedEvaluatorEventHandler.cs" /> + <Compile Include="Functional\Bridge\TestSecurityToken.cs" /> <Compile Include="Functional\Common\Task\ExceptionTask.cs" /> <Compile Include="Functional\Common\Task\NullTask.cs" /> <Compile Include="Functional\Failure\User\ContextStopExceptionTest.cs" /> @@ -236,6 +237,10 @@ under the License. <Project>{dec0f0a8-dbef-4ebf-b69c-e2369c15abf1}</Project> <Name>Org.Apache.REEF.IO</Name> </ProjectReference> + <ProjectReference Include="$(SolutionDir)\Org.Apache.REEF.Examples.HelloREEF\Org.Apache.REEF.Examples.HelloREEF.csproj"> + <Project>{0ff8cee9-b0b6-4a14-9a52-44441be048fe}</Project> + <Name>Org.Apache.REEF.Examples.HelloREEF</Name> + </ProjectReference> </ItemGroup> <ItemGroup> <Service Include="{82A7F48D-3B50-4B1E-B82E-3ADA8210C358}" /> @@ -244,4 +249,4 @@ under the License. <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" /> <Import Project="$(SolutionDir)\.nuget\NuGet.targets" Condition="Exists('$(SolutionDir)\.nuget\NuGet.targets')" /> <Import Project="$(PackagesDir)\StyleCop.MSBuild.$(StyleCopVersion)\build\StyleCop.MSBuild.Targets" Condition="Exists('$(PackagesDir)\StyleCop.MSBuild.$(StyleCopVersion)\build\StyleCop.MSBuild.Targets')" /> -</Project> +</Project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/reef/blob/85bae60a/lang/java/reef-bridge-client/pom.xml ---------------------------------------------------------------------- diff --git a/lang/java/reef-bridge-client/pom.xml b/lang/java/reef-bridge-client/pom.xml index 7621cc1..7bc6a28 100644 --- a/lang/java/reef-bridge-client/pom.xml +++ b/lang/java/reef-bridge-client/pom.xml @@ -132,6 +132,7 @@ under the License. <artifactId>avro-maven-plugin</artifactId> <executions> <execution> + <id>generate-local</id> <phase>generate-sources</phase> <goals> <goal>schema</goal> @@ -141,6 +142,17 @@ under the License. <outputDirectory>${project.basedir}/target/generated-sources/avro/</outputDirectory> </configuration> </execution> + <execution> + <id>generate-token-info</id> + <phase>generate-sources</phase> + <goals> + <goal>schema</goal> + </goals> + <configuration> + <sourceDirectory>${rootPath}/lang/common/token/avro/</sourceDirectory> + <outputDirectory>${project.basedir}/target/generated-sources/avro/</outputDirectory> + </configuration> + </execution> </executions> </plugin> <plugin> http://git-wip-us.apache.org/repos/asf/reef/blob/85bae60a/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/SecurityTokensReader.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/SecurityTokensReader.java b/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/SecurityTokensReader.java new file mode 100644 index 0000000..a1d908e --- /dev/null +++ b/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/SecurityTokensReader.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.bridge.client; + +import org.apache.avro.io.BinaryDecoder; +import org.apache.avro.io.DecoderFactory; +import org.apache.avro.io.DatumReader; +import org.apache.avro.specific.SpecificDatumReader; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.reef.runtime.common.files.REEFFileNames; + +import javax.inject.Inject; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Helper class to read security tokens from file and add them to the user's credentials. + * .Net SecurityTokenWriter stores the token info in a file and passes it to the java client. + * The method in this class will read this data and add tokens to UserGroupInformation. + */ +final class SecurityTokensReader { + + private static final Logger LOG = Logger.getLogger(SecurityTokensReader.class.getName()); + + private final DatumReader<SecurityToken> tokenDatumReader = new SpecificDatumReader<>(SecurityToken.class); + private final DecoderFactory decoderFactory = new DecoderFactory(); + private final File securityTokensFile; + + @Inject + private SecurityTokensReader(final REEFFileNames reefFileNames) { + this.securityTokensFile = new File(reefFileNames.getSecurityTokensFile()); + } + + /** + * Read tokens from a file and add them to the user's credentials. + * @param ugi user's credentials to add tokens to. + * @throws IOException if there are errors in reading the tokens' file. + */ + void addTokensFromFile(final UserGroupInformation ugi) throws IOException { + LOG.log(Level.FINE, "Reading security tokens from file: {0}", this.securityTokensFile); + + try (final FileInputStream stream = new FileInputStream(securityTokensFile)) { + final BinaryDecoder decoder = decoderFactory.binaryDecoder(stream, null); + + while (!decoder.isEnd()) { + final SecurityToken token = tokenDatumReader.read(null, decoder); + + final Token<TokenIdentifier> yarnToken = new Token<>( + token.getKey().array(), + token.getPassword().array(), + new Text(token.getKind().toString()), + new Text(token.getService().toString())); + + LOG.log(Level.FINE, "addToken for {0}", yarnToken.getKind()); + + ugi.addToken(yarnToken); + } + } + } +} http://git-wip-us.apache.org/repos/asf/reef/blob/85bae60a/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/YarnJobSubmissionClient.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/YarnJobSubmissionClient.java b/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/YarnJobSubmissionClient.java index fcf03ba..0d71517 100644 --- a/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/YarnJobSubmissionClient.java +++ b/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/YarnJobSubmissionClient.java @@ -18,6 +18,7 @@ */ package org.apache.reef.bridge.client; +import net.jcip.annotations.Immutable; import org.apache.commons.lang.Validate; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; @@ -25,6 +26,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -62,7 +64,7 @@ import java.util.logging.Logger; /** * The Java-side of the C# YARN Job Submission API. */ -@SuppressWarnings("checkstyle:hideutilityclassconstructor") +@Immutable public final class YarnJobSubmissionClient { private static final Logger LOG = Logger.getLogger(YarnJobSubmissionClient.class.getName()); @@ -76,17 +78,21 @@ public final class YarnJobSubmissionClient { private final YarnProxyUser yarnProxyUser; private final SecurityTokenProvider tokenProvider; private final YarnSubmissionParametersFileGenerator jobSubmissionParametersGenerator; + private final SecurityTokensReader securityTokensReader; + private static final String DEFAULT_TOKEN_KIND = "NULL"; @Inject - YarnJobSubmissionClient(@Parameter(DriverIsUnmanaged.class) final boolean isUnmanaged, - @Parameter(DriverLaunchCommandPrefix.class) final List<String> commandPrefixList, - final JobUploader uploader, - final YarnConfiguration yarnConfiguration, - final REEFFileNames fileNames, - final ClasspathProvider classpath, - final YarnProxyUser yarnProxyUser, - final SecurityTokenProvider tokenProvider, - final YarnSubmissionParametersFileGenerator jobSubmissionParametersGenerator) { + private YarnJobSubmissionClient( + @Parameter(DriverIsUnmanaged.class) final boolean isUnmanaged, + @Parameter(DriverLaunchCommandPrefix.class) final List<String> commandPrefixList, + final JobUploader uploader, + final YarnConfiguration yarnConfiguration, + final REEFFileNames fileNames, + final ClasspathProvider classpath, + final YarnProxyUser yarnProxyUser, + final SecurityTokenProvider tokenProvider, + final YarnSubmissionParametersFileGenerator jobSubmissionParametersGenerator, + final SecurityTokensReader securityTokensReader) { this.isUnmanaged = isUnmanaged; this.commandPrefixList = commandPrefixList; @@ -97,6 +103,7 @@ public final class YarnJobSubmissionClient { this.yarnProxyUser = yarnProxyUser; this.tokenProvider = tokenProvider; this.jobSubmissionParametersGenerator = jobSubmissionParametersGenerator; + this.securityTokensReader = securityTokensReader; } /** @@ -117,6 +124,9 @@ public final class YarnJobSubmissionClient { } private void launch(final YarnClusterSubmissionFromCS yarnSubmission) throws IOException, YarnException { + + this.setupCredentials(yarnSubmission); + // ------------------------------------------------------------------------ // Get an application ID try (final YarnSubmissionHelper submissionHelper = new YarnSubmissionHelper( @@ -167,20 +177,64 @@ public final class YarnJobSubmissionClient { } } - private static void writeSecurityTokenToUserCredential( + private void setupCredentials( final YarnClusterSubmissionFromCS yarnSubmission) throws IOException { - final UserGroupInformation currentUser = UserGroupInformation.getCurrentUser(); - final REEFFileNames fileNames = new REEFFileNames(); + + final File securityTokensFile = new File(fileNames.getSecurityTokensFile()); + + if (securityTokensFile.exists()) { + LOG.log(Level.INFO, "Writing security tokens to user credential"); + this.addTokensToCurrentUser(); + } else if (!yarnSubmission.getTokenKind().equalsIgnoreCase(DEFAULT_TOKEN_KIND)) { + // To support backward compatibility + LOG.log(Level.INFO, "Writing security token to user credential"); + this.writeSecurityTokenToUserCredential(yarnSubmission); + } else { + LOG.log(Level.FINE, "Did not find security token"); + } + } + + private void writeSecurityTokenToUserCredential( + final YarnClusterSubmissionFromCS yarnSubmission) throws IOException { + final String securityTokenIdentifierFile = fileNames.getSecurityTokenIdentifierFile(); final String securityTokenPasswordFile = fileNames.getSecurityTokenPasswordFile(); final Text tokenKind = new Text(yarnSubmission.getTokenKind()); final Text tokenService = new Text(yarnSubmission.getTokenService()); - byte[] identifier = Files.readAllBytes(Paths.get(securityTokenIdentifierFile)); - byte[] password = Files.readAllBytes(Paths.get(securityTokenPasswordFile)); - Token token = new Token(identifier, password, tokenKind, tokenService); + final byte[] identifier = Files.readAllBytes(Paths.get(securityTokenIdentifierFile)); + final byte[] password = Files.readAllBytes(Paths.get(securityTokenPasswordFile)); + + final Token<TokenIdentifier> token = new Token<>(identifier, password, tokenKind, tokenService); + + final UserGroupInformation currentUser = UserGroupInformation.getCurrentUser(); currentUser.addToken(token); } + private void addTokensToCurrentUser() throws IOException { + final UserGroupInformation currentUser = UserGroupInformation.getCurrentUser(); + logToken(Level.INFO, "Before adding client tokens,", currentUser); + this.securityTokensReader.addTokensFromFile(currentUser); + // Log info for debug purpose for now until it is stable then we can change the log level to FINE. + logToken(Level.INFO, "After adding client tokens,", currentUser); + } + + /** + * Log all the tokens in the container for thr user. + * + * @param logLevel - the log level. + * @param msgPrefix - msg prefix for log. + * @param user - the UserGroupInformation object. + */ + private static void logToken(final Level logLevel, final String msgPrefix, final UserGroupInformation user) { + if (LOG.isLoggable(logLevel)) { + LOG.log(logLevel, "{0} number of tokens: [{1}].", + new Object[] {msgPrefix, user.getCredentials().numberOfTokens()}); + for (final org.apache.hadoop.security.token.Token t : user.getCredentials().getAllTokens()) { + LOG.log(logLevel, "Token service: {0}, token kind: {1}.", new Object[] {t.getService(), t.getKind()}); + } + } + } + /** * We leave a file behind in job submission directory so that clr client can figure out * the applicationId and yarn rest endpoint. @@ -243,6 +297,8 @@ public final class YarnJobSubmissionClient { out.close(); } + private static final Tang TANG = Tang.Factory.getTang(); + /** * .NET client calls into this main method for job submission. * For arguments detail: @@ -265,14 +321,6 @@ public final class YarnJobSubmissionClient { appSubmissionParametersFile, jobSubmissionParametersFile); LOG.log(Level.INFO, "YARN job submission received from C#: {0}", yarnSubmission); - if (!yarnSubmission.getTokenKind().equalsIgnoreCase("NULL")) { - // We have to write security token to user credential before YarnJobSubmissionClient is created - // as that will need initialization of FileSystem which could need the token. - LOG.log(Level.INFO, "Writing security token to user credential"); - writeSecurityTokenToUserCredential(yarnSubmission); - } else{ - LOG.log(Level.FINE, "Did not find security token"); - } if (!yarnSubmission.getFileSystemUrl().equalsIgnoreCase(FileSystemUrl.DEFAULT_VALUE)) { LOG.log(Level.INFO, "getFileSystemUrl: {0}", yarnSubmission.getFileSystemUrl()); @@ -281,18 +329,20 @@ public final class YarnJobSubmissionClient { } final List<String> launchCommandPrefix = new ArrayList<String>() {{ - add(new REEFFileNames().getDriverLauncherExeFile().toString()); + final REEFFileNames reefFileNames = TANG.newInjector().getInstance(REEFFileNames.class); + add(reefFileNames.getDriverLauncherExeFile().getPath()); }}; - final Configuration yarnJobSubmissionClientConfig = Tang.Factory.getTang().newConfigurationBuilder() + final Configuration yarnJobSubmissionClientConfig = TANG.newConfigurationBuilder() .bindImplementation(RuntimeClasspathProvider.class, YarnClasspathProvider.class) .bindConstructor(org.apache.hadoop.yarn.conf.YarnConfiguration.class, YarnConfigurationConstructor.class) .bindNamedParameter(JobSubmissionDirectoryPrefix.class, yarnSubmission.getJobSubmissionDirectoryPrefix()) .bindNamedParameter(FileSystemUrl.class, yarnSubmission.getFileSystemUrl()) .bindList(DriverLaunchCommandPrefix.class, launchCommandPrefix) .build(); - final YarnJobSubmissionClient client = Tang.Factory.getTang().newInjector(yarnJobSubmissionClientConfig) - .getInstance(YarnJobSubmissionClient.class); + + final YarnJobSubmissionClient client = + TANG.newInjector(yarnJobSubmissionClientConfig).getInstance(YarnJobSubmissionClient.class); client.launch(yarnSubmission); http://git-wip-us.apache.org/repos/asf/reef/blob/85bae60a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/files/REEFFileNames.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/files/REEFFileNames.java b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/files/REEFFileNames.java index a51ad1c..811bbd4 100644 --- a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/files/REEFFileNames.java +++ b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/files/REEFFileNames.java @@ -49,16 +49,26 @@ public final class REEFFileNames { private static final String EVALUATOR_STDOUT = "evaluator.stdout"; private static final String DRIVER_HTTP_ENDPOINT_FILE_NAME = "DriverHttpEndpoint.txt"; private static final String BRIDGE_EXE_NAME = "Org.Apache.REEF.Bridge.exe"; - private static final String SECURITY_TOKEN_IDENTIFIER_FILE = "SecurityTokenId"; - private static final String SECURITY_TOKEN_PASSWORD_FILE = "SecurityTokenPwd"; + private static final String SECURITY_TOKENS_FILE = "SecurityTokens.json"; private static final String YARN_BOOTSTRAP_APP_PARAM_FILE = "yarn-app-parameters.json"; private static final String YARN_BOOTSTRAP_JOB_PARAM_FILE = "yarn-job-parameters.json"; + /** + * @deprecated TODO[JIRA REEF-1887] Use getSecurityTokensFile() instead. Remove in REEF 0.18. + */ + @Deprecated + private static final String SECURITY_TOKEN_IDENTIFIER_FILE = "SecurityTokenId"; + + /** + * @deprecated TODO[JIRA REEF-1887] Use getSecurityTokensFile() instead. Remove in REEF 0.18. + */ + @Deprecated + private static final String SECURITY_TOKEN_PASSWORD_FILE = "SecurityTokenPwd"; + @Inject public REEFFileNames() { } - /** * @return the filename of the CPP DLL for the bridge. */ @@ -222,19 +232,32 @@ public final class REEFFileNames { /** * @return File name that contains the security token identifier + * @deprecated TODO[JIRA REEF-1887] Use getSecurityTokensFile() instead. Remove in REEF 0.18. */ + @Deprecated public String getSecurityTokenIdentifierFile() { return SECURITY_TOKEN_IDENTIFIER_FILE; } /** * @return File name that contains the security token password + * @deprecated TODO[JIRA REEF-1887] Use getSecurityTokensFile() instead. Remove in REEF 0.18. */ + @Deprecated public String getSecurityTokenPasswordFile() { return SECURITY_TOKEN_PASSWORD_FILE; } /** + * @return Name of the file that contains security tokens and passwords. + * TODO[JIRA REEF-1887] This call supersedes getSecurityTokenIdentifierFile() + * and getSecurityTokenPasswordFile(). Remove this comment line in REEF 0.18. + */ + public String getSecurityTokensFile() { + return SECURITY_TOKENS_FILE; + } + + /** * @return File name the contains the bootstrap application parameters for YARN job submission * without Java dependency. */
