Repository: reef Updated Branches: refs/heads/master d5679922d -> 1afc6a607
[REEF-1019] Implement .NET REEF client using YARN REST APIs This addressed the issue by * Implementing IREEFClient using YARN REST submission * Create a temporary configuration to use this client * Creating a mode in HelloREEF to exercise this code * Use YARN commandline environment in LegacyJobResourceUploader * Minor cleanup of JobSubmissionDirectoryProvider JIRA: [REEF-1019](https://issues.apache.org/jira/browse/REEF-1019) Pull Request: Closes #770 Project: http://git-wip-us.apache.org/repos/asf/reef/repo Commit: http://git-wip-us.apache.org/repos/asf/reef/commit/1afc6a60 Tree: http://git-wip-us.apache.org/repos/asf/reef/tree/1afc6a60 Diff: http://git-wip-us.apache.org/repos/asf/reef/diff/1afc6a60 Branch: refs/heads/master Commit: 1afc6a6075f1063c57188f22dd951845eddad4ef Parents: d567992 Author: Anupam <[email protected]> Authored: Tue Nov 24 15:43:29 2015 -0800 Committer: Andrew Chung <[email protected]> Committed: Thu Jan 21 12:35:31 2016 -0800 ---------------------------------------------------------------------- .../JobResourceUploaderTests.cs | 21 +- .../LegacyJobResourceUploaderTests.cs | 28 +-- .../WindowsYarnJobCommandProviderTests.cs | 4 +- .../YarnClientTests.cs | 2 - .../YARN/AvroYarnJobSubmissionParameters.cs | 2 +- .../Org.Apache.REEF.Client.csproj | 3 +- .../YARN/IJobResourceUploader.cs | 3 +- .../YARN/IJobSubmissionDirectoryProvider.cs | 2 +- .../YARN/IYarnCommandLineEnvironment.cs | 2 +- .../YARN/JobSubmissionDirectoryProvider.cs | 23 +- .../YARN/LegacyJobResourceUploader.cs | 14 +- .../RESTClient/FileSystemJobResourceUploader.cs | 7 +- .../YARN/WindowsYarnJobCommandProvider.cs | 10 +- .../YARN/YARNClientConfiguration.cs | 10 + .../YARN/YarnCommandLineEnvironment.cs | 2 +- .../YARN/YarnREEFDotNetClient.cs | 228 +++++++++++++++++++ .../HelloREEF.cs | 5 +- .../src/main/avro/JobSubmissionParameters.avsc | 2 +- .../reef/bridge/client/JobResourceUploader.java | 15 +- ...SubmissionParametersSerializationFromCS.java | 5 +- 20 files changed, 305 insertions(+), 83 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/reef/blob/1afc6a60/lang/cs/Org.Apache.REEF.Client.Tests/JobResourceUploaderTests.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client.Tests/JobResourceUploaderTests.cs b/lang/cs/Org.Apache.REEF.Client.Tests/JobResourceUploaderTests.cs index 25b6fbf..66202ef 100644 --- a/lang/cs/Org.Apache.REEF.Client.Tests/JobResourceUploaderTests.cs +++ b/lang/cs/Org.Apache.REEF.Client.Tests/JobResourceUploaderTests.cs @@ -53,7 +53,7 @@ namespace Org.Apache.REEF.Client.Tests var testContext = new TestContext(); var jobResourceUploader = testContext.GetJobResourceUploader(); - jobResourceUploader.UploadJobResource(AnyDriverLocalFolderPath); + jobResourceUploader.UploadJobResource(AnyDriverLocalFolderPath, AnyDriverResourceUploadPath); // Archive file generator recieved exactly one call with correct driver local folder path testContext.ResourceArchiveFileGenerator.Received(1).CreateArchiveToUpload(AnyDriverLocalFolderPath); @@ -65,7 +65,7 @@ namespace Org.Apache.REEF.Client.Tests var testContext = new TestContext(); var jobResourceUploader = testContext.GetJobResourceUploader(); - var jobResource = jobResourceUploader.UploadJobResource(AnyDriverLocalFolderPath); + var jobResource = jobResourceUploader.UploadJobResource(AnyDriverLocalFolderPath, AnyDriverResourceUploadPath); Assert.AreEqual(AnyModificationTime, jobResource.LastModificationUnixTimestamp); Assert.AreEqual(AnyResourceSize, jobResource.ResourceSize); @@ -78,7 +78,7 @@ namespace Org.Apache.REEF.Client.Tests var testContext = new TestContext(); var jobResourceUploader = testContext.GetJobResourceUploader(); - jobResourceUploader.UploadJobResource(AnyDriverLocalFolderPath); + jobResourceUploader.UploadJobResource(AnyDriverLocalFolderPath, AnyDriverResourceUploadPath); testContext.FileSystem.Received(1).CreateUriForPath(AnyDriverResourceUploadPath); testContext.FileSystem.Received(1).CreateUriForPath(AnyUploadedResourcePath); @@ -88,21 +88,8 @@ namespace Org.Apache.REEF.Client.Tests .CreateDirectory(new Uri(AnyScheme + AnyHost + AnyDriverResourceUploadPath)); } - [TestMethod] - public void UploadJobResourceCallsJobSubmissionDirProvider() - { - var testContext = new TestContext(); - var jobResourceUploader = testContext.GetJobResourceUploader(); - - jobResourceUploader.UploadJobResource(AnyDriverLocalFolderPath); - - testContext.JobSubmissionDirectoryProvider.Received(1).GetJobSubmissionRemoteDirectory(); - } - private class TestContext { - public readonly IJobSubmissionDirectoryProvider JobSubmissionDirectoryProvider = - Substitute.For<IJobSubmissionDirectoryProvider>(); public readonly IResourceArchiveFileGenerator ResourceArchiveFileGenerator = Substitute.For<IResourceArchiveFileGenerator>(); public readonly IFileSystem FileSystem = Substitute.For<IFileSystem>(); @@ -110,7 +97,6 @@ namespace Org.Apache.REEF.Client.Tests public FileSystemJobResourceUploader GetJobResourceUploader() { var injector = TangFactory.GetTang().NewInjector(); - JobSubmissionDirectoryProvider.GetJobSubmissionRemoteDirectory().Returns(AnyDriverResourceUploadPath); FileSystem.GetFileStatus(new Uri(AnyUploadedResourceAbsoluteUri)) .Returns(new FileStatus(Epoch + TimeSpan.FromSeconds(AnyModificationTime), AnyResourceSize)); ResourceArchiveFileGenerator.CreateArchiveToUpload(AnyDriverLocalFolderPath) @@ -119,7 +105,6 @@ namespace Org.Apache.REEF.Client.Tests .Returns(new Uri(AnyScheme + AnyHost + AnyDriverResourceUploadPath)); FileSystem.CreateUriForPath(AnyUploadedResourcePath) .Returns(new Uri(AnyUploadedResourceAbsoluteUri)); - injector.BindVolatileInstance(GenericType<IJobSubmissionDirectoryProvider>.Class, JobSubmissionDirectoryProvider); injector.BindVolatileInstance(GenericType<IResourceArchiveFileGenerator>.Class, ResourceArchiveFileGenerator); injector.BindVolatileInstance(GenericType<IFileSystem>.Class, FileSystem); return injector.GetInstance<FileSystemJobResourceUploader>(); http://git-wip-us.apache.org/repos/asf/reef/blob/1afc6a60/lang/cs/Org.Apache.REEF.Client.Tests/LegacyJobResourceUploaderTests.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client.Tests/LegacyJobResourceUploaderTests.cs b/lang/cs/Org.Apache.REEF.Client.Tests/LegacyJobResourceUploaderTests.cs index 9b4e4f1..09fac49 100644 --- a/lang/cs/Org.Apache.REEF.Client.Tests/LegacyJobResourceUploaderTests.cs +++ b/lang/cs/Org.Apache.REEF.Client.Tests/LegacyJobResourceUploaderTests.cs @@ -36,18 +36,12 @@ namespace Org.Apache.REEF.Client.Tests private const long AnyResourceSize = 53092; [TestMethod] - public void LegacyJobResourceUploaderCanInstantiateWithDefaultBindings() - { - TangFactory.GetTang().NewInjector().GetInstance<LegacyJobResourceUploader>(); - } - - [TestMethod] public void UploadJobResourceCreatesResourceArchive() { var testContext = new TestContext(); var jobResourceUploader = testContext.GetJobResourceUploader(); - jobResourceUploader.UploadJobResource(AnyDriverLocalFolderPath); + jobResourceUploader.UploadJobResource(AnyDriverLocalFolderPath, AnyDriverResourceUploadPath); // Archive file generator recieved exactly one call with correct driver local folder path with trailing \ testContext.ResourceArchiveFileGenerator.Received(1).CreateArchiveToUpload(AnyDriverLocalFolderPath + @"\"); @@ -61,8 +55,7 @@ namespace Org.Apache.REEF.Client.Tests const string anyLocalArchivePath = @"Any\Local\Archive\Path.zip"; testContext.ResourceArchiveFileGenerator.CreateArchiveToUpload(AnyDriverLocalFolderPath + @"\") .Returns(anyLocalArchivePath); - testContext.JobSubmissionDirectoryProvider.GetJobSubmissionRemoteDirectory().Returns(AnyDriverResourceUploadPath); - jobResourceUploader.UploadJobResource(AnyDriverLocalFolderPath); + jobResourceUploader.UploadJobResource(AnyDriverLocalFolderPath, AnyDriverResourceUploadPath); const string javaClassNameForResourceUploader = @"org.apache.reef.bridge.client.JobResourceUploader"; Guid notUsed; @@ -86,7 +79,7 @@ namespace Org.Apache.REEF.Client.Tests var jobResourceUploader = testContext.GetJobResourceUploader(fileExistsReturnValue: false); // throws filenotfound exception - jobResourceUploader.UploadJobResource(AnyDriverLocalFolderPath); + jobResourceUploader.UploadJobResource(AnyDriverLocalFolderPath, AnyDriverResourceUploadPath); } [TestMethod] @@ -95,7 +88,7 @@ namespace Org.Apache.REEF.Client.Tests var testContext = new TestContext(); var jobResourceUploader = testContext.GetJobResourceUploader(); - var jobResource = jobResourceUploader.UploadJobResource(AnyDriverLocalFolderPath); + var jobResource = jobResourceUploader.UploadJobResource(AnyDriverLocalFolderPath, AnyDriverResourceUploadPath); Assert.AreEqual(AnyModificationTime, jobResource.LastModificationUnixTimestamp); Assert.AreEqual(AnyResourceSize, jobResource.ResourceSize); @@ -105,11 +98,8 @@ namespace Org.Apache.REEF.Client.Tests private class TestContext { public readonly IJavaClientLauncher JavaClientLauncher = Substitute.For<IJavaClientLauncher>(); - public readonly IJobSubmissionDirectoryProvider JobSubmissionDirectoryProvider = - Substitute.For<IJobSubmissionDirectoryProvider>(); public readonly IResourceArchiveFileGenerator ResourceArchiveFileGenerator = Substitute.For<IResourceArchiveFileGenerator>(); - public readonly IFile File = Substitute.For<IFile>(); public LegacyJobResourceUploader GetJobResourceUploader(bool fileExistsReturnValue = true, string uploadedResourcePath = AnyUploadedResourcePath, @@ -117,13 +107,15 @@ namespace Org.Apache.REEF.Client.Tests long resourceSize = AnyResourceSize) { var injector = TangFactory.GetTang().NewInjector(); - File.Exists(Arg.Any<string>()).Returns(fileExistsReturnValue); - File.ReadAllText(Arg.Any<string>()) + IFile file = Substitute.For<IFile>(); + IYarnCommandLineEnvironment yarn = Substitute.For<IYarnCommandLineEnvironment>(); + file.Exists(Arg.Any<string>()).Returns(fileExistsReturnValue); + file.ReadAllText(Arg.Any<string>()) .Returns(string.Format("{0};{1};{2}", uploadedResourcePath, modificationTime, resourceSize)); injector.BindVolatileInstance(GenericType<IJavaClientLauncher>.Class, JavaClientLauncher); - injector.BindVolatileInstance(GenericType<IJobSubmissionDirectoryProvider>.Class, JobSubmissionDirectoryProvider); injector.BindVolatileInstance(GenericType<IResourceArchiveFileGenerator>.Class, ResourceArchiveFileGenerator); - injector.BindVolatileInstance(GenericType<IFile>.Class, File); + injector.BindVolatileInstance(GenericType<IFile>.Class, file); + injector.BindVolatileInstance(GenericType<IYarnCommandLineEnvironment>.Class, yarn); return injector.GetInstance<LegacyJobResourceUploader>(); } } http://git-wip-us.apache.org/repos/asf/reef/blob/1afc6a60/lang/cs/Org.Apache.REEF.Client.Tests/WindowsYarnJobCommandProviderTests.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client.Tests/WindowsYarnJobCommandProviderTests.cs b/lang/cs/Org.Apache.REEF.Client.Tests/WindowsYarnJobCommandProviderTests.cs index ca4af15..f93d225 100644 --- a/lang/cs/Org.Apache.REEF.Client.Tests/WindowsYarnJobCommandProviderTests.cs +++ b/lang/cs/Org.Apache.REEF.Client.Tests/WindowsYarnJobCommandProviderTests.cs @@ -50,9 +50,7 @@ namespace Org.Apache.REEF.Client.Tests "%HADOOP_HOME%/share/hadoop/hdfs/*", "%HADOOP_HOME%/share/hadoop/hdfs/lib/*", "%HADOOP_HOME%/share/hadoop/mapreduce/*", - "%HADOOP_HOME%/share/hadoop/mapreduce/lib/*", - "reef/local/*", - "reef/global/*" + "%HADOOP_HOME%/share/hadoop/mapreduce/lib/*" }; [TestMethod] http://git-wip-us.apache.org/repos/asf/reef/blob/1afc6a60/lang/cs/Org.Apache.REEF.Client.Tests/YarnClientTests.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client.Tests/YarnClientTests.cs b/lang/cs/Org.Apache.REEF.Client.Tests/YarnClientTests.cs index 0af520b..986ac5e 100644 --- a/lang/cs/Org.Apache.REEF.Client.Tests/YarnClientTests.cs +++ b/lang/cs/Org.Apache.REEF.Client.Tests/YarnClientTests.cs @@ -17,14 +17,12 @@ using System; using System.Collections.Generic; -using System.IO; using System.Linq; using System.Net; using System.Threading; using System.Threading.Tasks; using Microsoft.VisualStudio.TestTools.UnitTesting; using NSubstitute; -using Org.Apache.REEF.Client.Yarn; using Org.Apache.REEF.Client.Yarn.RestClient; using Org.Apache.REEF.Client.YARN.RestClient; using Org.Apache.REEF.Client.YARN.RestClient.DataModel; http://git-wip-us.apache.org/repos/asf/reef/blob/1afc6a60/lang/cs/Org.Apache.REEF.Client/Avro/YARN/AvroYarnJobSubmissionParameters.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/Avro/YARN/AvroYarnJobSubmissionParameters.cs b/lang/cs/Org.Apache.REEF.Client/Avro/YARN/AvroYarnJobSubmissionParameters.cs index 7f856ac..36c5806 100644 --- a/lang/cs/Org.Apache.REEF.Client/Avro/YARN/AvroYarnJobSubmissionParameters.cs +++ b/lang/cs/Org.Apache.REEF.Client/Avro/YARN/AvroYarnJobSubmissionParameters.cs @@ -28,7 +28,7 @@ namespace Org.Apache.REEF.Client.Avro.YARN [DataContract(Namespace = "org.apache.reef.reef.bridge.client.avro")] public sealed class AvroYarnJobSubmissionParameters { - private const string JsonSchema = @"{""type"":""record"",""name"":""org.apache.reef.reef.bridge.client.avro.AvroYarnJobSubmissionParameters"",""doc"":""General cross-language submission parameters to the YARN runtime"",""fields"":[{""name"":""sharedJobSubmissionParameters"",""type"":{""type"":""record"",""name"":""org.apache.reef.reef.bridge.client.avro.AvroJobSubmissionParameters"",""doc"":""General cross-language submission parameters shared by all runtimes"",""fields"":[{""name"":""jobId"",""type"":""string""},{""name"":""tcpBeginPort"",""type"":""int""},{""name"":""tcpRangeCount"",""type"":""int""},{""name"":""tcpTryCount"",""type"":""int""},{""name"":""jobSubmissionFolder"",""type"":""string""}]}},{""name"":""driverMemory"",""type"":""int""},{""name"":""driverRecoveryTimeout"",""type"":""int""},{""name"":""dfsJobSubmissionFolder"",""type"":[""null"",""string""]},{""name"":""jobSubmissionDirectoryPrefix"",""type"":""string""}]}"; + private const string JsonSchema = @"{""type"":""record"",""name"":""org.apache.reef.reef.bridge.client.avro.AvroYarnJobSubmissionParameters"",""doc"":""General cross-language submission parameters to the YARN runtime"",""fields"":[{""name"":""sharedJobSubmissionParameters"",""type"":{""type"":""record"",""name"":""org.apache.reef.reef.bridge.client.avro.AvroJobSubmissionParameters"",""doc"":""General cross-language submission parameters shared by all runtimes"",""fields"":[{""name"":""jobId"",""type"":""string""},{""name"":""tcpBeginPort"",""type"":""int""},{""name"":""tcpRangeCount"",""type"":""int""},{""name"":""tcpTryCount"",""type"":""int""},{""name"":""jobSubmissionFolder"",""type"":""string""}]}},{""name"":""driverMemory"",""type"":""int""},{""name"":""driverRecoveryTimeout"",""type"":""int""},{""name"":""dfsJobSubmissionFolder"",""type"":""string"",""default"":""NULL""},{""name"":""jobSubmissionDirectoryPrefix"",""type"":""string""}]}"; /// <summary> /// Gets the schema. http://git-wip-us.apache.org/repos/asf/reef/blob/1afc6a60/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj b/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj index 0fda57b..d44b14d 100644 --- a/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj +++ b/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj @@ -142,6 +142,7 @@ under the License. <Compile Include="YARN\RestClient\YarnRestAPIException.cs" /> <Compile Include="YARN\YARNClientConfiguration.cs" /> <Compile Include="YARN\YarnCommandLineEnvironment.cs" /> + <Compile Include="YARN\YarnREEFDotNetClient.cs" /> </ItemGroup> <ItemGroup> <None Include="Org.Apache.REEF.Client.nuspec" /> @@ -302,4 +303,4 @@ under the License. </Target> <Target Name="BeforeBuild" DependsOnTargets="$(BeforeBuildDependsOn);RewriteClientResources"> </Target> -</Project> +</Project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/reef/blob/1afc6a60/lang/cs/Org.Apache.REEF.Client/YARN/IJobResourceUploader.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/IJobResourceUploader.cs b/lang/cs/Org.Apache.REEF.Client/YARN/IJobResourceUploader.cs index 2366f01..b4b1640 100644 --- a/lang/cs/Org.Apache.REEF.Client/YARN/IJobResourceUploader.cs +++ b/lang/cs/Org.Apache.REEF.Client/YARN/IJobResourceUploader.cs @@ -26,7 +26,8 @@ namespace Org.Apache.REEF.Client.Yarn /// Upload archived local driver folder to DFS destination path. /// </summary> /// <param name="driverLocalFolderPath">Local folder where REEF application resources are staged</param> + /// <param name="jobSubmissionDirectory">Remote directory path where we will upload resources</param> /// <returns>Path, modification time and size of uploaded file as JobResource</returns> - JobResource UploadJobResource(string driverLocalFolderPath); + JobResource UploadJobResource(string driverLocalFolderPath, string jobSubmissionDirectory); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/reef/blob/1afc6a60/lang/cs/Org.Apache.REEF.Client/YARN/IJobSubmissionDirectoryProvider.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/IJobSubmissionDirectoryProvider.cs b/lang/cs/Org.Apache.REEF.Client/YARN/IJobSubmissionDirectoryProvider.cs index 3faa62d..c6ef7a2 100644 --- a/lang/cs/Org.Apache.REEF.Client/YARN/IJobSubmissionDirectoryProvider.cs +++ b/lang/cs/Org.Apache.REEF.Client/YARN/IJobSubmissionDirectoryProvider.cs @@ -29,6 +29,6 @@ namespace Org.Apache.REEF.Client.Yarn /// Returns path to job submission directory in DFS. /// </summary> /// <returns></returns> - string GetJobSubmissionRemoteDirectory(); + string GetJobSubmissionRemoteDirectory(string applicationId); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/reef/blob/1afc6a60/lang/cs/Org.Apache.REEF.Client/YARN/IYarnCommandLineEnvironment.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/IYarnCommandLineEnvironment.cs b/lang/cs/Org.Apache.REEF.Client/YARN/IYarnCommandLineEnvironment.cs index 2990155..9695ddf 100644 --- a/lang/cs/Org.Apache.REEF.Client/YARN/IYarnCommandLineEnvironment.cs +++ b/lang/cs/Org.Apache.REEF.Client/YARN/IYarnCommandLineEnvironment.cs @@ -30,6 +30,6 @@ namespace Org.Apache.REEF.Client.Yarn /// Returns the class path returned by `yarn classpath`. /// </summary> /// <returns>The class path returned by `yarn classpath`.</returns> - IList<string> GetYarnClasspathList(); + IReadOnlyList<string> GetYarnClasspathList(); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/reef/blob/1afc6a60/lang/cs/Org.Apache.REEF.Client/YARN/JobSubmissionDirectoryProvider.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/JobSubmissionDirectoryProvider.cs b/lang/cs/Org.Apache.REEF.Client/YARN/JobSubmissionDirectoryProvider.cs index fe1b007..aa7f896 100644 --- a/lang/cs/Org.Apache.REEF.Client/YARN/JobSubmissionDirectoryProvider.cs +++ b/lang/cs/Org.Apache.REEF.Client/YARN/JobSubmissionDirectoryProvider.cs @@ -15,34 +15,39 @@ // specific language governing permissions and limitations // under the License. -using System; using Org.Apache.REEF.Client.YARN.Parameters; using Org.Apache.REEF.Common.Files; using Org.Apache.REEF.Tang.Annotations; namespace Org.Apache.REEF.Client.Yarn { - public class JobSubmissionDirectoryProvider : IJobSubmissionDirectoryProvider + /// <summary> + /// Provides path to job submission directory. + /// </summary> + public sealed class JobSubmissionDirectoryProvider : IJobSubmissionDirectoryProvider { private readonly string _jobSubmissionDirectoryPrefix; - private readonly REEFFileNames _filenames; + private readonly REEFFileNames _fileNames; [Inject] private JobSubmissionDirectoryProvider( - [Parameter(typeof(JobSubmissionDirectoryPrefixParameter))] + [Parameter(typeof(JobSubmissionDirectoryPrefixParameter))] string jobSubmissionDirectoryPrefix, - REEFFileNames filenames) + REEFFileNames fileNames) { - _filenames = filenames; + _fileNames = fileNames; _jobSubmissionDirectoryPrefix = jobSubmissionDirectoryPrefix; } - public string GetJobSubmissionRemoteDirectory() + /// <summary> + /// Returns path to job submission directory in DFS. + /// </summary> + public string GetJobSubmissionRemoteDirectory(string applicationId) { return string.Format(@"{0}/{1}{2}/", _jobSubmissionDirectoryPrefix, - DateTime.Now.ToString("yyyy_MM_dd_HH_mm_ss"), - _filenames.GetJobFolderPrefix()); + _fileNames.GetJobFolderPrefix(), + applicationId); } } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/reef/blob/1afc6a60/lang/cs/Org.Apache.REEF.Client/YARN/LegacyJobResourceUploader.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/LegacyJobResourceUploader.cs b/lang/cs/Org.Apache.REEF.Client/YARN/LegacyJobResourceUploader.cs index 0c01709..5fde77d 100644 --- a/lang/cs/Org.Apache.REEF.Client/YARN/LegacyJobResourceUploader.cs +++ b/lang/cs/Org.Apache.REEF.Client/YARN/LegacyJobResourceUploader.cs @@ -40,26 +40,26 @@ namespace Org.Apache.REEF.Client.Yarn @"org.apache.reef.bridge.client.JobResourceUploader"; private readonly IJavaClientLauncher _javaLauncher; - private readonly IJobSubmissionDirectoryProvider _jobSubmissionDirectoryProvider; private readonly IResourceArchiveFileGenerator _resourceArchiveFileGenerator; private readonly IFile _file; [Inject] - private LegacyJobResourceUploader(IJavaClientLauncher javaLauncher, - IJobSubmissionDirectoryProvider jobSubmissionDirectoryProvider, + private LegacyJobResourceUploader( + IJavaClientLauncher javaLauncher, IResourceArchiveFileGenerator resourceArchiveFileGenerator, - IFile file) + IFile file, + IYarnCommandLineEnvironment yarn) { _file = file; _resourceArchiveFileGenerator = resourceArchiveFileGenerator; - _jobSubmissionDirectoryProvider = jobSubmissionDirectoryProvider; _javaLauncher = javaLauncher; + _javaLauncher.AddToClassPath(yarn.GetYarnClasspathList()); } - public JobResource UploadJobResource(string driverLocalFolderPath) + public JobResource UploadJobResource(string driverLocalFolderPath, string jobSubmissionDirectory) { driverLocalFolderPath = driverLocalFolderPath.TrimEnd('\\') + @"\"; - string driverUploadPath = _jobSubmissionDirectoryProvider.GetJobSubmissionRemoteDirectory().TrimEnd('/') + @"/"; + string driverUploadPath = jobSubmissionDirectory.TrimEnd('/') + @"/"; Log.Log(Level.Info, "DriverFolderPath: {0} DriverUploadPath: {1}", driverLocalFolderPath, driverUploadPath); var archivePath = _resourceArchiveFileGenerator.CreateArchiveToUpload(driverLocalFolderPath); http://git-wip-us.apache.org/repos/asf/reef/blob/1afc6a60/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/FileSystemJobResourceUploader.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/FileSystemJobResourceUploader.cs b/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/FileSystemJobResourceUploader.cs index df83509..bcff6d2 100644 --- a/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/FileSystemJobResourceUploader.cs +++ b/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/FileSystemJobResourceUploader.cs @@ -34,25 +34,22 @@ namespace Org.Apache.REEF.Client.YARN.RestClient { private static readonly Logger Log = Logger.GetLogger(typeof(FileSystemJobResourceUploader)); private static readonly DateTime Epoch = new DateTime(1970, 1, 1, 0, 0, 0, 0); - private readonly IJobSubmissionDirectoryProvider _jobSubmissionDirectoryProvider; private readonly IResourceArchiveFileGenerator _resourceArchiveFileGenerator; private readonly IFileSystem _fileSystem; [Inject] private FileSystemJobResourceUploader( - IJobSubmissionDirectoryProvider jobSubmissionDirectoryProvider, IResourceArchiveFileGenerator resourceArchiveFileGenerator, IFileSystem fileSystem) { _fileSystem = fileSystem; _resourceArchiveFileGenerator = resourceArchiveFileGenerator; - _jobSubmissionDirectoryProvider = jobSubmissionDirectoryProvider; } - public JobResource UploadJobResource(string driverLocalFolderPath) + public JobResource UploadJobResource(string driverLocalFolderPath, string jobSubmissionDirectory) { driverLocalFolderPath = driverLocalFolderPath.TrimEnd('\\') + @"\"; - var driverUploadPath = _jobSubmissionDirectoryProvider.GetJobSubmissionRemoteDirectory().TrimEnd('/') + @"/"; + var driverUploadPath = jobSubmissionDirectory.TrimEnd('/') + @"/"; Log.Log(Level.Verbose, "DriverFolderPath: {0} DriverUploadPath: {1}", driverLocalFolderPath, driverUploadPath); var archivePath = _resourceArchiveFileGenerator.CreateArchiveToUpload(driverLocalFolderPath); http://git-wip-us.apache.org/repos/asf/reef/blob/1afc6a60/lang/cs/Org.Apache.REEF.Client/YARN/WindowsYarnJobCommandProvider.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/WindowsYarnJobCommandProvider.cs b/lang/cs/Org.Apache.REEF.Client/YARN/WindowsYarnJobCommandProvider.cs index c7d280d..6a14c5c 100644 --- a/lang/cs/Org.Apache.REEF.Client/YARN/WindowsYarnJobCommandProvider.cs +++ b/lang/cs/Org.Apache.REEF.Client/YARN/WindowsYarnJobCommandProvider.cs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +using System.Collections.Generic; using System.Text; using Org.Apache.REEF.Client.Yarn; using Org.Apache.REEF.Client.YARN.Parameters; @@ -75,7 +76,14 @@ namespace Org.Apache.REEF.Client.YARN sb.Append(" " + string.Format(JvmOptionsMaxMemoryAllocationPoolSizeFormat, _driverMaxMemoryAllocationPoolSizeMB)); sb.Append(" " + ClassPathToken); - sb.Append(" " + string.Join(";", _yarnCommandLineEnvironment.GetYarnClasspathList())); + + var yarnClasspathList = new List<string>(_yarnCommandLineEnvironment.GetYarnClasspathList()) + { + string.Format("{0}/{1}/*", _fileNames.GetReefFolderName(), _fileNames.GetLocalFolderName()), + string.Format("{0}/{1}/*", _fileNames.GetReefFolderName(), _fileNames.GetGlobalFolderName()) + }; + + sb.Append(" " + string.Join(";", yarnClasspathList)); sb.Append(" " + ProcReefProperty); if (_enableDebugLogging) { http://git-wip-us.apache.org/repos/asf/reef/blob/1afc6a60/lang/cs/Org.Apache.REEF.Client/YARN/YARNClientConfiguration.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/YARNClientConfiguration.cs b/lang/cs/Org.Apache.REEF.Client/YARN/YARNClientConfiguration.cs index d14203f..0920cb7 100644 --- a/lang/cs/Org.Apache.REEF.Client/YARN/YARNClientConfiguration.cs +++ b/lang/cs/Org.Apache.REEF.Client/YARN/YARNClientConfiguration.cs @@ -20,6 +20,7 @@ using Org.Apache.REEF.Client.YARN; using Org.Apache.REEF.Client.YARN.Parameters; using Org.Apache.REEF.Tang.Formats; using Org.Apache.REEF.Tang.Util; +using Org.Apache.REEF.Utilities.Attributes; namespace Org.Apache.REEF.Client.Yarn { @@ -38,5 +39,14 @@ namespace Org.Apache.REEF.Client.Yarn .BindNamedParameter(GenericType<SecurityTokenKindParameter>.Class, SecurityTokenKind) .BindNamedParameter(GenericType<SecurityTokenServiceParameter>.Class, SecurityTokenService) .Build(); + + [Unstable("This is temporary configuration until REEF-70 is completed when ConfigurationModule" + + " and ConfigurationModuleYARNRest would be merged.")] + public static ConfigurationModule ConfigurationModuleYARNRest = new YARNClientConfiguration() + .BindImplementation(GenericType<IREEFClient>.Class, GenericType<YarnREEFDotNetClient>.Class) + .BindNamedParameter(GenericType<JobSubmissionDirectoryPrefixParameter>.Class, JobSubmissionFolderPrefix) + .BindNamedParameter(GenericType<SecurityTokenKindParameter>.Class, SecurityTokenKind) + .BindNamedParameter(GenericType<SecurityTokenServiceParameter>.Class, SecurityTokenService) + .Build(); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/reef/blob/1afc6a60/lang/cs/Org.Apache.REEF.Client/YARN/YarnCommandLineEnvironment.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/YarnCommandLineEnvironment.cs b/lang/cs/Org.Apache.REEF.Client/YARN/YarnCommandLineEnvironment.cs index 6593cdd..39dd789 100644 --- a/lang/cs/Org.Apache.REEF.Client/YARN/YarnCommandLineEnvironment.cs +++ b/lang/cs/Org.Apache.REEF.Client/YARN/YarnCommandLineEnvironment.cs @@ -43,7 +43,7 @@ namespace Org.Apache.REEF.Client.Yarn /// Returns the class path returned by `yarn classpath`. /// </summary> /// <returns>The class path returned by `yarn classpath`.</returns> - public IList<string> GetYarnClasspathList() + public IReadOnlyList<string> GetYarnClasspathList() { return Yarn("classpath").Split(';').Distinct().ToList(); } http://git-wip-us.apache.org/repos/asf/reef/blob/1afc6a60/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFDotNetClient.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFDotNetClient.cs b/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFDotNetClient.cs new file mode 100644 index 0000000..89dad2b --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFDotNetClient.cs @@ -0,0 +1,228 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Org.Apache.REEF.Client.API; +using Org.Apache.REEF.Client.Avro; +using Org.Apache.REEF.Client.Avro.YARN; +using Org.Apache.REEF.Client.Common; +using Org.Apache.REEF.Client.Yarn; +using Org.Apache.REEF.Client.Yarn.RestClient; +using Org.Apache.REEF.Client.YARN.RestClient.DataModel; +using Org.Apache.REEF.Common.Avro; +using Org.Apache.REEF.Common.Files; +using Org.Apache.REEF.Driver.Bridge; +using Org.Apache.REEF.Tang.Annotations; +using Org.Apache.REEF.Tang.Implementations.Tang; +using Org.Apache.REEF.Utilities.Attributes; +using Org.Apache.REEF.Utilities.Logging; +using Org.Apache.REEF.Wake.Remote.Parameters; + +namespace Org.Apache.REEF.Client.YARN +{ + /// <summary> + /// Temporary client for developing and testing .NET job submission E2E. + /// TODO: When REEF-189 is completed YARNREEFClient should be either merged or + /// deprecated by final client. + /// </summary> + [Unstable("For security token support we still need to use YARNREEFClient until (REEF-875)")] + public sealed class YarnREEFDotNetClient : IREEFClient + { + private const string REEFApplicationType = @"REEF"; + private static readonly Logger Log = Logger.GetLogger(typeof(YarnREEFDotNetClient)); + private readonly IYarnRMClient _yarnRMClient; + private readonly DriverFolderPreparationHelper _driverFolderPreparationHelper; + private readonly IJobResourceUploader _jobResourceUploader; + private readonly IYarnJobCommandProvider _yarnJobCommandProvider; + private readonly REEFFileNames _fileNames; + private readonly IJobSubmissionDirectoryProvider _jobSubmissionDirectoryProvider; + + [Inject] + private YarnREEFDotNetClient( + IYarnRMClient yarnRMClient, + DriverFolderPreparationHelper driverFolderPreparationHelper, + IJobResourceUploader jobResourceUploader, + IYarnJobCommandProvider yarnJobCommandProvider, + REEFFileNames fileNames, + IJobSubmissionDirectoryProvider jobSubmissionDirectoryProvider) + { + _jobSubmissionDirectoryProvider = jobSubmissionDirectoryProvider; + _fileNames = fileNames; + _yarnJobCommandProvider = yarnJobCommandProvider; + _jobResourceUploader = jobResourceUploader; + _driverFolderPreparationHelper = driverFolderPreparationHelper; + _yarnRMClient = yarnRMClient; + } + + public void Submit(IJobSubmission jobSubmission) + { + string jobId = jobSubmission.JobIdentifier; + + // todo: Future client interface should be async. + // Using GetAwaiter().GetResult() instead of .Result to avoid exception + // getting wrapped in AggregateException. + var newApplication = _yarnRMClient.CreateNewApplicationAsync().GetAwaiter().GetResult(); + string applicationId = newApplication.ApplicationId; + + // create job submission remote path + string jobSubmissionDirectory = + _jobSubmissionDirectoryProvider.GetJobSubmissionRemoteDirectory(applicationId); + + // create local driver folder. + var localDriverFolderPath = CreateDriverFolder(jobId, applicationId); + try + { + Log.Log(Level.Verbose, "Preparing driver folder in {0}", localDriverFolderPath); + _driverFolderPreparationHelper.PrepareDriverFolder(jobSubmission, localDriverFolderPath); + + // prepare configuration + var paramInjector = TangFactory.GetTang().NewInjector(jobSubmission.DriverConfigurations.ToArray()); + int maxApplicationSubmissions = + paramInjector.GetNamedInstance<DriverBridgeConfigurationOptions.MaxApplicationSubmissions, int>(); + + var avroJobSubmissionParameters = new AvroJobSubmissionParameters + { + jobId = jobSubmission.JobIdentifier, + tcpBeginPort = paramInjector.GetNamedInstance<TcpPortRangeStart, int>(), + tcpRangeCount = paramInjector.GetNamedInstance<TcpPortRangeCount, int>(), + tcpTryCount = paramInjector.GetNamedInstance<TcpPortRangeTryCount, int>(), + jobSubmissionFolder = localDriverFolderPath + }; + + var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters + { + driverMemory = jobSubmission.DriverMemory, + driverRecoveryTimeout = + paramInjector.GetNamedInstance<DriverBridgeConfigurationOptions.DriverRestartEvaluatorRecoverySeconds, int>(), + jobSubmissionDirectoryPrefix = jobSubmissionDirectory, + dfsJobSubmissionFolder = jobSubmissionDirectory, + sharedJobSubmissionParameters = avroJobSubmissionParameters + }; + + var submissionArgsFilePath = Path.Combine(localDriverFolderPath, + _fileNames.GetLocalFolderPath(), + _fileNames.GetJobSubmissionParametersFile()); + using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew)) + { + var serializedArgs = + AvroJsonSerializer<AvroYarnJobSubmissionParameters>.ToBytes(avroYarnJobSubmissionParameters); + argsFileStream.Write(serializedArgs, 0, serializedArgs.Length); + } + + // upload prepared folder to DFS + var jobResource = _jobResourceUploader.UploadJobResource(localDriverFolderPath, jobSubmissionDirectory); + + // submit job + Log.Log(Level.Verbose, @"Assigned application id {0}", applicationId); + + var submissionReq = CreateApplicationSubmissionRequest(jobSubmission, + applicationId, + maxApplicationSubmissions, + jobResource); + var submittedApplication = _yarnRMClient.SubmitApplicationAsync(submissionReq).GetAwaiter().GetResult(); + Log.Log(Level.Info, @"Submitted application {0}", submittedApplication.Id); + } + finally + { + if (Directory.Exists(localDriverFolderPath)) + { + Directory.Delete(localDriverFolderPath, recursive: true); + } + } + } + + public IJobSubmissionResult SubmitAndGetJobStatus(IJobSubmission jobSubmission) + { + throw new NotSupportedException(); + } + + public async Task<FinalState> GetJobFinalStatus(string appId) + { + var application = await _yarnRMClient.GetApplicationAsync(appId).ConfigureAwait(false); + return application.FinalStatus; + } + + private SubmitApplication CreateApplicationSubmissionRequest( + IJobSubmission jobSubmission, + string appId, + int maxApplicationSubmissions, + JobResource jobResource) + { + string command = _yarnJobCommandProvider.GetJobSubmissionCommand(); + Log.Log(Level.Verbose, "Command for YARN: {0}", command); + Log.Log(Level.Verbose, "ApplicationID: {0}", appId); + Log.Log(Level.Verbose, "MaxApplicationSubmissions: {0}", maxApplicationSubmissions); + Log.Log(Level.Verbose, "Driver archive location: {0}", jobResource.RemoteUploadPath); + + var submitApplication = new SubmitApplication + { + ApplicationId = appId, + ApplicationName = jobSubmission.JobIdentifier, + AmResource = new Resouce + { + MemoryMB = jobSubmission.DriverMemory, + VCores = 1 // keeping parity with existing code + }, + MaxAppAttempts = maxApplicationSubmissions, + ApplicationType = REEFApplicationType, + KeepContainersAcrossApplicationAttempts = true, + Queue = @"default", // keeping parity with existing code + Priority = 1, // keeping parity with existing code + UnmanagedAM = false, + AmContainerSpec = new AmContainerSpec + { + LocalResources = new LocalResources + { + Entries = new[] + { + new KeyValuePair<string, LocalResourcesValue> + { + Key = _fileNames.GetReefFolderName(), + Value = new LocalResourcesValue + { + Resource = jobResource.RemoteUploadPath, + Type = ResourceType.ARCHIVE, + Visibility = Visibility.APPLICATION, + Size = jobResource.ResourceSize, + Timestamp = jobResource.LastModificationUnixTimestamp + } + } + } + }, + Commands = new Commands + { + Command = command + } + } + }; + + return submitApplication; + } + + /// <summary> + /// Creates the temporary directory to hold the job submission. + /// </summary> + /// <returns>The path to the folder created.</returns> + private string CreateDriverFolder(string jobId, string appId) + { + return Path.GetFullPath(Path.Combine(Path.GetTempPath(), string.Join("-", "reef", jobId, appId))); + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/reef/blob/1afc6a60/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs index b402577..0a3ed65 100644 --- a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs +++ b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs @@ -36,6 +36,7 @@ namespace Org.Apache.REEF.Examples.HelloREEF private static readonly Logger Logger = Logger.GetLogger(typeof(HelloREEF)); private const string Local = "local"; private const string YARN = "yarn"; + private const string YARNRest = "yarnrest"; private readonly IREEFClient _reefClient; private readonly JobSubmissionBuilderFactory _jobSubmissionBuilderFactory; @@ -64,7 +65,7 @@ namespace Org.Apache.REEF.Examples.HelloREEF .SetJobIdentifier("HelloREEF") .Build(); - _reefClient.SubmitAndGetJobStatus(helloJobSubmission); + _reefClient.Submit(helloJobSubmission); } /// <summary> @@ -81,6 +82,8 @@ namespace Org.Apache.REEF.Examples.HelloREEF .Build(); case YARN: return YARNClientConfiguration.ConfigurationModule.Build(); + case YARNRest: + return YARNClientConfiguration.ConfigurationModuleYARNRest.Build(); default: throw new Exception("Unknown runtime: " + name); } http://git-wip-us.apache.org/repos/asf/reef/blob/1afc6a60/lang/java/reef-bridge-client/src/main/avro/JobSubmissionParameters.avsc ---------------------------------------------------------------------- diff --git a/lang/java/reef-bridge-client/src/main/avro/JobSubmissionParameters.avsc b/lang/java/reef-bridge-client/src/main/avro/JobSubmissionParameters.avsc index ef06e25..fec2431 100644 --- a/lang/java/reef-bridge-client/src/main/avro/JobSubmissionParameters.avsc +++ b/lang/java/reef-bridge-client/src/main/avro/JobSubmissionParameters.avsc @@ -49,7 +49,7 @@ { "name": "sharedJobSubmissionParameters", "type": "AvroJobSubmissionParameters" }, { "name": "driverMemory", "type": "int" }, { "name": "driverRecoveryTimeout", "type": "int" }, - { "name": "dfsJobSubmissionFolder", "type": ["null", "string"], "default" : null }, + { "name": "dfsJobSubmissionFolder", "type": "string", "default": "NULL" }, { "name": "jobSubmissionDirectoryPrefix", "type": "string" } ] }, http://git-wip-us.apache.org/repos/asf/reef/blob/1afc6a60/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/JobResourceUploader.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/JobResourceUploader.java b/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/JobResourceUploader.java index 1e9fb25..6ad77c7 100644 --- a/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/JobResourceUploader.java +++ b/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/JobResourceUploader.java @@ -21,11 +21,11 @@ package org.apache.reef.bridge.client; import org.apache.commons.lang.Validate; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.URL; -import org.apache.reef.runtime.yarn.client.YarnClientConfiguration; +import org.apache.reef.runtime.common.files.RuntimeClasspathProvider; +import org.apache.reef.runtime.yarn.YarnClasspathProvider; import org.apache.reef.runtime.yarn.client.uploader.JobUploader; -import org.apache.reef.runtime.yarn.driver.parameters.JobSubmissionDirectoryPrefix; +import org.apache.reef.runtime.yarn.util.YarnConfigurationConstructor; import org.apache.reef.tang.Configuration; -import org.apache.reef.tang.Configurations; import org.apache.reef.tang.Tang; import org.apache.reef.tang.exceptions.InjectionException; @@ -57,11 +57,10 @@ public final class JobResourceUploader { LOG.log(Level.INFO, "Received args: LocalPath " + localFile.getAbsolutePath() + " Submission directory " + jobSubmissionDirectory + " LocalOutputPath " + localOutputPath); - final Configuration configuration = Configurations.merge( - Tang.Factory.getTang().newConfigurationBuilder() - .bindNamedParameter(JobSubmissionDirectoryPrefix.class, jobSubmissionDirectory) - .build(), - YarnClientConfiguration.CONF.build()); + final Configuration configuration = Tang.Factory.getTang().newConfigurationBuilder() + .bindImplementation(RuntimeClasspathProvider.class, YarnClasspathProvider.class) + .bindConstructor(org.apache.hadoop.yarn.conf.YarnConfiguration.class, YarnConfigurationConstructor.class) + .build(); final JobUploader jobUploader = Tang.Factory.getTang() .newInjector(configuration) http://git-wip-us.apache.org/repos/asf/reef/blob/1afc6a60/lang/java/reef-bridge-client/src/test/java/org/apache/reef/bridge/client/TestAvroJobSubmissionParametersSerializationFromCS.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-bridge-client/src/test/java/org/apache/reef/bridge/client/TestAvroJobSubmissionParametersSerializationFromCS.java b/lang/java/reef-bridge-client/src/test/java/org/apache/reef/bridge/client/TestAvroJobSubmissionParametersSerializationFromCS.java index ba930d4..c79fe41 100644 --- a/lang/java/reef-bridge-client/src/test/java/org/apache/reef/bridge/client/TestAvroJobSubmissionParametersSerializationFromCS.java +++ b/lang/java/reef-bridge-client/src/test/java/org/apache/reef/bridge/client/TestAvroJobSubmissionParametersSerializationFromCS.java @@ -57,10 +57,7 @@ public final class TestAvroJobSubmissionParametersSerializationFromCS { "}," + "\"driverMemory\":" + NUMBER_REP + "," + "\"driverRecoveryTimeout\":" + NUMBER_REP + "," + - "\"dfsJobSubmissionFolder\":" + - "{" + - "\"string\": " + STRING_REP_QUOTED + - "}," + + "\"dfsJobSubmissionFolder\":\"" + STRING_REP + "\"," + "\"jobSubmissionDirectoryPrefix\":" + STRING_REP_QUOTED + "}";
