[REEF-1204] Interface separation between Job and Application Parameters This addressed the issue by * Deprecate IJobSubmission and its related classes and interfaces. * Added JobRequest, JobParameters, AppParameters, and their related classes. * Added methods to submit jobs with the newly introduced classes.
JIRA: [REEF-1204](https://issues.apache.org/jira/browse/REEF-1204) Pull Request: Closes #844 Project: http://git-wip-us.apache.org/repos/asf/reef/repo Commit: http://git-wip-us.apache.org/repos/asf/reef/commit/4b758fe2 Tree: http://git-wip-us.apache.org/repos/asf/reef/tree/4b758fe2 Diff: http://git-wip-us.apache.org/repos/asf/reef/diff/4b758fe2 Branch: refs/heads/master Commit: 4b758fe2059ba998a8bbf69cb8d5dea8eb5a8148 Parents: f4d40ca Author: Andrew Chung <[email protected]> Authored: Wed Feb 17 13:29:19 2016 -0800 Committer: Anupam <[email protected]> Committed: Fri Feb 19 20:16:10 2016 -0800 ---------------------------------------------------------------------- .../YarnREEFParamSerializerTests.cs | 21 +-- .../Org.Apache.REEF.Client/API/AppParameters.cs | 110 ++++++++++++ .../API/AppParametersBuilder.cs | 161 +++++++++++++++++ .../API/IJobSubmission.cs | 2 + .../API/IJobSubmissionBuilder.cs | 1 + .../Org.Apache.REEF.Client/API/IREEFClient.cs | 15 +- .../Org.Apache.REEF.Client/API/JobParameters.cs | 72 ++++++++ .../API/JobParametersBuilder.cs | 80 +++++++++ .../cs/Org.Apache.REEF.Client/API/JobRequest.cs | 135 ++++++++++++++ .../API/JobRequestBuilder.cs | 176 +++++++++++++++++++ .../Org.Apache.REEF.Client/API/JobSubmission.cs | 29 ++- .../API/JobSubmissionBuilder.cs | 56 ++---- .../API/JobSubmissionBuilderFactory.cs | 2 + .../Common/DriverFolderPreparationHelper.cs | 28 +-- .../Org.Apache.REEF.Client/Common/FileSets.cs | 2 +- .../Org.Apache.REEF.Client/Local/LocalClient.cs | 37 ++-- .../Org.Apache.REEF.Client.csproj | 6 + .../YARN/YARNREEFClient.cs | 33 ++-- .../YARN/YarnREEFDotNetClient.cs | 33 ++-- .../YARN/YarnREEFDotNetParamSerializer.cs | 14 +- .../YARN/YarnREEFParamSerializer.cs | 20 +-- .../AllHandlers.cs | 8 +- .../DriverRestart.cs | 8 +- .../HelloREEF.cs | 10 +- .../OnREEF/Client/REEFIMRUClient.cs | 12 +- .../BroadcastAndReduceClient.cs | 4 +- .../Functional/ReefFunctionalTest.cs | 4 +- 27 files changed, 908 insertions(+), 171 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/reef/blob/4b758fe2/lang/cs/Org.Apache.REEF.Client.Tests/YarnREEFParamSerializerTests.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client.Tests/YarnREEFParamSerializerTests.cs b/lang/cs/Org.Apache.REEF.Client.Tests/YarnREEFParamSerializerTests.cs index 3b0b8fa..16f5533 100644 --- a/lang/cs/Org.Apache.REEF.Client.Tests/YarnREEFParamSerializerTests.cs +++ b/lang/cs/Org.Apache.REEF.Client.Tests/YarnREEFParamSerializerTests.cs @@ -62,10 +62,10 @@ namespace Org.Apache.REEF.Client.Tests var injector = TangFactory.GetTang().NewInjector(tcpConf, driverConf); var serializer = injector.GetInstance<YarnREEFDotNetParamSerializer>(); - var jobSubmission = injector.GetInstance<JobSubmissionBuilderFactory>() - .GetJobSubmissionBuilder().Build(); + var jobRequest = injector.GetInstance<JobRequestBuilder>().Build(); - var serializedBytes = serializer.SerializeAppArgsToBytes(jobSubmission, injector, AnyString); + var serializedBytes = serializer.SerializeAppArgsToBytes( + jobRequest.AppParameters, injector, AnyString); var expectedString = Encoding.UTF8.GetString(serializedBytes); var jsonObject = JObject.Parse(expectedString); var expectedJsonObject = JObject.Parse(expectedJson); @@ -90,10 +90,9 @@ namespace Org.Apache.REEF.Client.Tests var injector = TangFactory.GetTang().NewInjector(); var serializer = injector.GetInstance<YarnREEFDotNetParamSerializer>(); - var jobSubmission = injector.GetInstance<JobSubmissionBuilderFactory>() - .GetJobSubmissionBuilder().SetJobIdentifier(AnyString).Build(); + var jobRequest = injector.GetInstance<JobRequestBuilder>().SetJobIdentifier(AnyString).Build(); - var serializedBytes = serializer.SerializeJobArgsToBytes(jobSubmission, AnyString, AnyString); + var serializedBytes = serializer.SerializeJobArgsToBytes(jobRequest.JobParameters, AnyString, AnyString); var expectedString = Encoding.UTF8.GetString(serializedBytes); var jsonObject = JObject.Parse(expectedString); var expectedJsonObject = JObject.Parse(expectedJson); @@ -127,13 +126,12 @@ namespace Org.Apache.REEF.Client.Tests var injector = TangFactory.GetTang().NewInjector(tcpConf, driverConf); var serializer = injector.GetInstance<YarnREEFParamSerializer>(); - var jobSubmission = injector.GetInstance<JobSubmissionBuilderFactory>() - .GetJobSubmissionBuilder() + var jobRequest = injector.GetInstance<JobRequestBuilder>() .SetDriverMemory(AnyInt) .SetMaxApplicationSubmissions(AnyInt) .Build(); - var serializedBytes = serializer.SerializeAppArgsToBytes(jobSubmission, injector); + var serializedBytes = serializer.SerializeAppArgsToBytes(jobRequest.AppParameters, injector); var expectedString = Encoding.UTF8.GetString(serializedBytes); var jsonObject = JObject.Parse(expectedString); var expectedJsonObject = JObject.Parse(expectedJson); @@ -171,14 +169,13 @@ namespace Org.Apache.REEF.Client.Tests var injector = TangFactory.GetTang().NewInjector(conf); var serializer = injector.GetInstance<YarnREEFParamSerializer>(); - var jobSubmission = injector.GetInstance<JobSubmissionBuilderFactory>() - .GetJobSubmissionBuilder() + var jobRequest = injector.GetInstance<JobRequestBuilder>() .SetJobIdentifier(AnyString) .SetMaxApplicationSubmissions(AnyInt) .SetDriverMemory(AnyInt) .Build(); - var serializedBytes = serializer.SerializeJobArgsToBytes(jobSubmission, injector, AnyString); + var serializedBytes = serializer.SerializeJobArgsToBytes(jobRequest.JobParameters, injector, AnyString); var expectedString = Encoding.UTF8.GetString(serializedBytes); var jsonObject = JObject.Parse(expectedString); var expectedJsonObject = JObject.Parse(expectedJson); http://git-wip-us.apache.org/repos/asf/reef/blob/4b758fe2/lang/cs/Org.Apache.REEF.Client/API/AppParameters.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/API/AppParameters.cs b/lang/cs/Org.Apache.REEF.Client/API/AppParameters.cs new file mode 100644 index 0000000..9841811 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Client/API/AppParameters.cs @@ -0,0 +1,110 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System; +using System.Collections.Generic; +using Org.Apache.REEF.Tang.Interface; + +namespace Org.Apache.REEF.Client.API +{ + /// <summary> + /// The parameters for a REEF application, used to specify application parameters for each REEF application. + /// For job parameters which is specified each time for job submissions of the same + /// REEF application, see <see cref="JobParameters"/>. + /// </summary> + public sealed class AppParameters + { + private readonly ISet<IConfiguration> _driverConfigurations; + private readonly ISet<string> _globalAssemblies; + private readonly ISet<string> _globalFiles; + private readonly ISet<string> _localAssemblies; + private readonly ISet<string> _localFiles; + private readonly string _driverConfigurationFileContents; + + internal AppParameters( + ISet<IConfiguration> driverConfigurations, + ISet<string> globalAssemblies, + ISet<string> globalFiles, + ISet<string> localAssemblies, + ISet<string> localFiles, + string driverConfigurationFileContents) + { + _driverConfigurations = driverConfigurations; + _globalAssemblies = globalAssemblies; + _globalFiles = globalFiles; + _localAssemblies = localAssemblies; + _localFiles = localFiles; + _driverConfigurationFileContents = driverConfigurationFileContents; + } + + [Obsolete("Introduced to bridge deprecation of IJobSubmission.")] + internal static AppParameters FromJobSubmission(IJobSubmission jobSubmission) + { + return new AppParameters(jobSubmission.DriverConfigurations, jobSubmission.GlobalAssemblies, jobSubmission.GlobalFiles, + jobSubmission.LocalAssemblies, jobSubmission.LocalFiles, jobSubmission.DriverConfigurationFileContents); + } + + /// <summary> + /// The assemblies to be made available to all containers. + /// </summary> + public ISet<string> GlobalAssemblies + { + get { return _globalAssemblies; } + } + + /// <summary> + /// The driver configurations. + /// </summary> + public ISet<IConfiguration> DriverConfigurations + { + get { return _driverConfigurations; } + } + + /// <summary> + /// The global files to be made available to all containers. + /// </summary> + public ISet<string> GlobalFiles + { + get { return _globalFiles; } + } + + /// <summary> + /// The assemblies to be made available only to the local container. + /// </summary> + public ISet<string> LocalAssemblies + { + get { return _localAssemblies; } + } + + /// <summary> + /// The files to be made available only to the local container. + /// </summary> + public ISet<string> LocalFiles + { + get { return _localFiles; } + } + + /// <summary> + /// Driver config file contents (Org.Apache.REEF.Bridge.exe.config) + /// Can be use to redirect assembly versions + /// </summary> + public string DriverConfigurationFileContents + { + get { return _driverConfigurationFileContents; } + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/reef/blob/4b758fe2/lang/cs/Org.Apache.REEF.Client/API/AppParametersBuilder.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/API/AppParametersBuilder.cs b/lang/cs/Org.Apache.REEF.Client/API/AppParametersBuilder.cs new file mode 100644 index 0000000..0b12c5e --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Client/API/AppParametersBuilder.cs @@ -0,0 +1,161 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System; +using System.Collections.Generic; +using System.IO; +using Org.Apache.REEF.Tang.Interface; + +namespace Org.Apache.REEF.Client.API +{ + /// <summary> + /// A builder for <see cref="AppParameters"/>. + /// </summary> + public sealed class AppParametersBuilder + { + private readonly ISet<IConfiguration> _driverConfigurations = new HashSet<IConfiguration>(); + private readonly ISet<string> _globalAssemblies = new HashSet<string>(); + private readonly ISet<string> _globalFiles = new HashSet<string>(); + private readonly ISet<string> _localAssemblies = new HashSet<string>(); + private readonly ISet<string> _localFiles = new HashSet<string>(); + private readonly ISet<IConfigurationProvider> _configurationProviders = new HashSet<IConfigurationProvider>(); + + private string _driverConfigurationFileContents; + + private AppParametersBuilder() + { + } + + /// <summary> + /// Creates a new <see cref="AppParametersBuilder"/>. + /// </summary> + public static AppParametersBuilder NewBuilder() + { + return new AppParametersBuilder(); + } + + /// <summary> + /// Add a file to be made available in all containers. + /// </summary> + /// <param name="fileName"></param> + /// <returns></returns> + public AppParametersBuilder AddGlobalFile(string fileName) + { + _globalFiles.Add(fileName); + return this; + } + + /// <summary> + /// Add a file to be made available only on the driver. + /// </summary> + /// <param name="fileName"></param> + /// <returns></returns> + public AppParametersBuilder AddLocalFile(string fileName) + { + _localFiles.Add(fileName); + return this; + } + + /// <summary> + /// Add an assembly to be made available on all containers. + /// </summary> + /// <param name="fileName"></param> + /// <returns></returns> + public AppParametersBuilder AddGlobalAssembly(string fileName) + { + _globalAssemblies.Add(fileName); + return this; + } + + /// <summary> + /// Add an assembly to the driver only. + /// </summary> + /// <param name="fileName"></param> + /// <returns></returns> + public AppParametersBuilder AddLocalAssembly(string fileName) + { + _localAssemblies.Add(fileName); + return this; + } + + /// <summary> + /// Add a Configuration to the Driver. + /// </summary> + public AppParametersBuilder AddDriverConfiguration(IConfiguration configuration) + { + _driverConfigurations.Add(configuration); + return this; + } + + /// <summary> + /// Adds a driver configuration provider for configurations of the Driver. + /// </summary> + public AppParametersBuilder AddDriverConfigurationProviders( + IEnumerable<IConfigurationProvider> configurationProviders) + { + _configurationProviders.UnionWith(configurationProviders); + return this; + } + + /// <summary> + /// Add the assembly needed for the given Type to the driver. + /// </summary> + public AppParametersBuilder AddLocalAssemblyForType(Type type) + { + AddLocalAssembly(GetAssemblyPathForType(type)); + return this; + } + + /// <summary> + /// Add the assembly needed for the given Type to all containers. + /// </summary> + public AppParametersBuilder AddGlobalAssemblyForType(Type type) + { + AddGlobalAssembly(GetAssemblyPathForType(type)); + return this; + } + + /// <summary> + /// Driver config file contents (Org.Apache.REEF.Bridge.exe.config) contents + /// Can be use to redirect assembly versions + /// </summary> + /// <param name="driverConfigurationFileContents">Driver configuration file contents.</param> + public AppParametersBuilder SetDriverConfigurationFileContents(string driverConfigurationFileContents) + { + _driverConfigurationFileContents = driverConfigurationFileContents; + return this; + } + + /// <summary> + /// Finds the path to the assembly the given Type was loaded from. + /// </summary> + private static string GetAssemblyPathForType(Type type) + { + var path = Uri.UnescapeDataString(new UriBuilder(type.Assembly.CodeBase).Path); + return Path.GetFullPath(path); + } + + /// <summary> + /// Builds the application parameters. + /// </summary> + public AppParameters Build() + { + return new AppParameters(_driverConfigurations, _globalAssemblies, _globalFiles, _localAssemblies, + _localFiles, _driverConfigurationFileContents); + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/reef/blob/4b758fe2/lang/cs/Org.Apache.REEF.Client/API/IJobSubmission.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/API/IJobSubmission.cs b/lang/cs/Org.Apache.REEF.Client/API/IJobSubmission.cs index dac9023..5cefb98 100644 --- a/lang/cs/Org.Apache.REEF.Client/API/IJobSubmission.cs +++ b/lang/cs/Org.Apache.REEF.Client/API/IJobSubmission.cs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +using System; using System.Collections.Generic; using Org.Apache.REEF.Tang.Interface; @@ -24,6 +25,7 @@ namespace Org.Apache.REEF.Client.API /// This interfaces provides all the information that is needed for /// a job submission /// </summary> + [Obsolete("Deprecated in 0.14. Please use JobRequest.")] public interface IJobSubmission { /// <summary> http://git-wip-us.apache.org/repos/asf/reef/blob/4b758fe2/lang/cs/Org.Apache.REEF.Client/API/IJobSubmissionBuilder.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/API/IJobSubmissionBuilder.cs b/lang/cs/Org.Apache.REEF.Client/API/IJobSubmissionBuilder.cs index 4a35b41..4e2408b 100644 --- a/lang/cs/Org.Apache.REEF.Client/API/IJobSubmissionBuilder.cs +++ b/lang/cs/Org.Apache.REEF.Client/API/IJobSubmissionBuilder.cs @@ -23,6 +23,7 @@ namespace Org.Apache.REEF.Client.API /// <summary> /// Facilitates building of job submissions /// </summary> + [Obsolete("Deprecated in 0.14, please use JobRequestBuilder.")] public interface IJobSubmissionBuilder { /// <summary> http://git-wip-us.apache.org/repos/asf/reef/blob/4b758fe2/lang/cs/Org.Apache.REEF.Client/API/IREEFClient.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/API/IREEFClient.cs b/lang/cs/Org.Apache.REEF.Client/API/IREEFClient.cs index 2337e00..355ca70 100644 --- a/lang/cs/Org.Apache.REEF.Client/API/IREEFClient.cs +++ b/lang/cs/Org.Apache.REEF.Client/API/IREEFClient.cs @@ -30,19 +30,26 @@ namespace Org.Apache.REEF.Client.API public interface IREEFClient { /// <summary> - /// Submit the job described in jobSubmission to the cluster. + /// Submit the job described in jobRequest to the cluster. /// </summary> /// <param name="jobSubmission"></param> + [Obsolete("Deprecated in 0.14. Please use Submit(JobRequest)")] void Submit(IJobSubmission jobSubmission); /// <summary> - /// Submit the job described in jobSubmission to the cluster. + /// Submit the job described in jobRequest to the cluster. + /// </summary> + /// <param name="jobRequest"></param> + void Submit(JobRequest jobRequest); + + /// <summary> + /// Submit the job described in jobRequest to the cluster. /// Expect IJobSubmissionResult returned after the call. /// </summary> - /// <param name="jobSubmission"></param> + /// <param name="jobRequest"></param> /// <returns>IJobSubmissionResult</returns> [Unstable("0.13", "Working in progress for what to return after submit")] - IJobSubmissionResult SubmitAndGetJobStatus(IJobSubmission jobSubmission); + IJobSubmissionResult SubmitAndGetJobStatus(JobRequest jobRequest); /// <summary> /// Returns the application status in running the job http://git-wip-us.apache.org/repos/asf/reef/blob/4b758fe2/lang/cs/Org.Apache.REEF.Client/API/JobParameters.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/API/JobParameters.cs b/lang/cs/Org.Apache.REEF.Client/API/JobParameters.cs new file mode 100644 index 0000000..ef5bed6 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Client/API/JobParameters.cs @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System; + +namespace Org.Apache.REEF.Client.API +{ + /// <summary> + /// The parameters for a REEF job, used to specify job parameters on each REEF submission. + /// For application parameters which is specified only once for all job submissions of the same + /// REEF application, see <see cref="AppParameters"/>. + /// </summary> + public sealed class JobParameters + { + private readonly string _jobIdentifier; + private readonly int _maxApplicationSubmissions; + private readonly int _driverMemory; + + internal JobParameters(string jobIdentifier, int maxApplicationSubmissions, int driverMemory) + { + _jobIdentifier = jobIdentifier; + _maxApplicationSubmissions = maxApplicationSubmissions; + _driverMemory = driverMemory; + } + + [Obsolete("Introduced to bridge deprecation of IJobSubmission.")] + internal static JobParameters FromJobSubmission(IJobSubmission jobSubmission) + { + return new JobParameters( + jobSubmission.JobIdentifier, jobSubmission.MaxApplicationSubmissions, jobSubmission.DriverMemory); + } + + /// <summary> + /// The identifier of the job. + /// </summary> + public string JobIdentifier + { + get { return _jobIdentifier; } + } + + /// <summary> + /// The maximum amount of times the job can be submitted. Used primarily in the + /// driver restart scenario. + /// </summary> + public int MaxApplicationSubmissions + { + get { return _maxApplicationSubmissions; } + } + + /// <summary> + /// The size of the driver memory, in MB. + /// </summary> + public int DriverMemoryInMB + { + get { return _driverMemory; } + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/reef/blob/4b758fe2/lang/cs/Org.Apache.REEF.Client/API/JobParametersBuilder.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/API/JobParametersBuilder.cs b/lang/cs/Org.Apache.REEF.Client/API/JobParametersBuilder.cs new file mode 100644 index 0000000..f960ee7 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Client/API/JobParametersBuilder.cs @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Org.Apache.REEF.Client.API +{ + /// <summary> + /// A builder for <see cref="JobParameters"/>. + /// </summary> + public sealed class JobParametersBuilder + { + private string _jobIdentifier; + private int _maxApplicationSubmissions = 1; + private int _driverMemory = 512; + + private JobParametersBuilder() + { + } + + /// <summary> + /// Creates a new <see cref="JobParametersBuilder"/>. + /// </summary> + public static JobParametersBuilder NewBuilder() + { + return new JobParametersBuilder(); + } + + /// <summary> + /// Builds a <see cref="JobParameters"/> object based on parameters passed to the builder. + /// </summary> + /// <returns></returns> + public JobParameters Build() + { + return new JobParameters(_jobIdentifier, _maxApplicationSubmissions, _driverMemory); + } + + /// <summary> + /// Sets the identifier of the job. + /// </summary> + public JobParametersBuilder SetJobIdentifier(string id) + { + _jobIdentifier = id; + return this; + } + + /// <summary> + /// Sets the maximum amount of times the job can be submitted. Used primarily in the + /// driver restart scenario. + /// </summary> + public JobParametersBuilder SetMaxApplicationSubmissions(int maxApplicationSubmissions) + { + _maxApplicationSubmissions = maxApplicationSubmissions; + return this; + } + + /// <summary> + /// Sets the amount of memory (in MB) to allocate for the Driver. + /// </summary> + /// <param name="driverMemoryInMb">The amount of memory (in MB) to allocate for the Driver.</param> + /// <returns>this</returns> + public JobParametersBuilder SetDriverMemory(int driverMemoryInMb) + { + _driverMemory = driverMemoryInMb; + return this; + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/reef/blob/4b758fe2/lang/cs/Org.Apache.REEF.Client/API/JobRequest.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/API/JobRequest.cs b/lang/cs/Org.Apache.REEF.Client/API/JobRequest.cs new file mode 100644 index 0000000..2e3dc6d --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Client/API/JobRequest.cs @@ -0,0 +1,135 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System; +using System.Collections.Generic; +using Org.Apache.REEF.Tang.Interface; + +namespace Org.Apache.REEF.Client.API +{ + /// <summary> + /// Describes parameters for a single job submission. + /// </summary> + public sealed class JobRequest + { + private readonly JobParameters _jobParameters; + private readonly AppParameters _appParameters; + + internal JobRequest(JobParameters jobParameters, AppParameters appParameters) + { + _jobParameters = jobParameters; + _appParameters = appParameters; + } + + [Obsolete("Introduced to bridge deprecation of IJobSubmission.")] + internal static JobRequest FromJobSubmission(IJobSubmission jobSubmission) + { + return new JobRequest( + JobParameters.FromJobSubmission(jobSubmission), AppParameters.FromJobSubmission(jobSubmission)); + } + + /// <summary> + /// The assemblies to be made available to all containers. + /// </summary> + public ISet<string> GlobalAssemblies + { + get { return _appParameters.GlobalAssemblies; } + } + + /// <summary> + /// The driver configurations + /// </summary> + public ISet<IConfiguration> DriverConfigurations + { + get { return _appParameters.DriverConfigurations; } + } + + /// <summary> + /// The global files to be made available to all containers. + /// </summary> + public ISet<string> GlobalFiles + { + get { return _appParameters.GlobalFiles; } + } + + /// <summary> + /// The assemblies to be made available only to the local container. + /// </summary> + public ISet<string> LocalAssemblies + { + get { return _appParameters.LocalAssemblies; } + } + + /// <summary> + /// The files to be made available only to the local container. + /// </summary> + public ISet<string> LocalFiles + { + get { return _appParameters.LocalFiles; } + } + + /// <summary> + /// The size of the driver memory, in MB. + /// </summary> + public int DriverMemory + { + get { return _jobParameters.DriverMemoryInMB; } + } + + /// <summary> + /// The maximum amount of times the job can be submitted. Used primarily in the + /// driver restart scenario. + /// </summary> + public int MaxApplicationSubmissions + { + get { return _jobParameters.MaxApplicationSubmissions; } + } + + /// <summary> + /// The Job's identifier + /// </summary> + public string JobIdentifier + { + get { return _jobParameters.JobIdentifier; } + } + + /// <summary> + /// Driver config file contents (Org.Apache.REEF.Bridge.exe.config) + /// Can be use to redirect assembly versions + /// </summary> + public string DriverConfigurationFileContents + { + get { return _appParameters.DriverConfigurationFileContents; } + } + + /// <summary> + /// Gets the <see cref="JobParameters"/> for this particular job submission. + /// </summary> + public JobParameters JobParameters + { + get { return _jobParameters; } + } + + /// <summary> + /// Gets the <see cref="AppParameters"/> for this particular job submission. + /// </summary> + public AppParameters AppParameters + { + get { return _appParameters; } + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/reef/blob/4b758fe2/lang/cs/Org.Apache.REEF.Client/API/JobRequestBuilder.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/API/JobRequestBuilder.cs b/lang/cs/Org.Apache.REEF.Client/API/JobRequestBuilder.cs new file mode 100644 index 0000000..9bef241 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Client/API/JobRequestBuilder.cs @@ -0,0 +1,176 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System; +using System.Collections.Generic; +using Org.Apache.REEF.Common.Client.Parameters; +using Org.Apache.REEF.Tang.Annotations; +using Org.Apache.REEF.Tang.Interface; + +namespace Org.Apache.REEF.Client.API +{ + public sealed class JobRequestBuilder + { + private readonly AppParametersBuilder _appParametersBuilder = AppParametersBuilder.NewBuilder(); + private readonly JobParametersBuilder _jobParametersBuilder = JobParametersBuilder.NewBuilder(); + + private JobRequestBuilder() + { + } + + [Inject] + private JobRequestBuilder([Parameter(typeof(DriverConfigurationProviders))] ISet<IConfigurationProvider> configurationProviders) + { + AddDriverConfigurationProviders(configurationProviders); + } + + public static JobRequestBuilder NewBuilder() + { + return new JobRequestBuilder(); + } + + /// <summary> + /// Bake the information provided so far and return a IJobSubmission + /// </summary> + public JobRequest Build() + { + return new JobRequest(_jobParametersBuilder.Build(), _appParametersBuilder.Build()); + } + + /// <summary> + /// Make this file available to all containers + /// </summary> + public JobRequestBuilder AddGlobalFile(string fileName) + { + _appParametersBuilder.AddGlobalFile(fileName); + return this; + } + + /// <summary> + /// Files specific to one container + /// </summary> + public JobRequestBuilder AddLocalFile(string fileName) + { + _appParametersBuilder.AddLocalFile(fileName); + return this; + } + + /// <summary> + /// Add an assembly to be made available on all containers. + /// </summary> + /// <param name="fileName"></param> + /// <returns></returns> + public JobRequestBuilder AddGlobalAssembly(string fileName) + { + _appParametersBuilder.AddGlobalAssembly(fileName); + return this; + } + + /// <summary> + /// Add an assembly to the driver only. + /// </summary> + /// <param name="fileName"></param> + /// <returns></returns> + public JobRequestBuilder AddLocalAssembly(string fileName) + { + _appParametersBuilder.AddLocalAssembly(fileName); + return this; + } + + /// <summary> + /// Add a Configuration to the Driver. + /// </summary> + /// <param name="configuration"></param> + /// <returns></returns> + public JobRequestBuilder AddDriverConfiguration(IConfiguration configuration) + { + _appParametersBuilder.AddDriverConfiguration(configuration); + return this; + } + + /// <summary> + /// Add the assembly needed for the given Type to the driver. + /// </summary> + /// <param name="type"></param> + /// <returns></returns> + public JobRequestBuilder AddLocalAssemblyForType(Type type) + { + _appParametersBuilder.AddLocalAssemblyForType(type); + return this; + } + + /// <summary> + /// Add the assembly needed for the given Type to all containers. + /// </summary> + /// <param name="type"></param> + /// <returns></returns> + public JobRequestBuilder AddGlobalAssemblyForType(Type type) + { + _appParametersBuilder.AddGlobalAssemblyForType(type); + return this; + } + + /// <summary> + /// Gives the job an identifier. + /// </summary> + /// <param name="id"></param> + /// <returns></returns> + public JobRequestBuilder SetJobIdentifier(string id) + { + _jobParametersBuilder.SetJobIdentifier(id); + return this; + } + + /// <summary> + /// Sets the amount of memory (in MB) to allocate for the Driver. + /// </summary> + /// <param name="driverMemoryInMb">The amount of memory (in MB) to allocate for the Driver.</param> + /// <returns>this</returns> + public JobRequestBuilder SetDriverMemory(int driverMemoryInMb) + { + _jobParametersBuilder.SetDriverMemory(driverMemoryInMb); + return this; + } + + /// <summary> + /// Sets the maximum amount of times a job can be submitted. + /// </summary> + public JobRequestBuilder SetMaxApplicationSubmissions(int maxAppSubmissions) + { + _jobParametersBuilder.SetMaxApplicationSubmissions(maxAppSubmissions); + return this; + } + + /// <summary> + /// Driver config file contents (Org.Apache.REEF.Bridge.exe.config) contents + /// Can be use to redirect assembly versions + /// </summary> + /// <param name="driverConfigurationFileContents">Driver configuration file contents.</param> + /// <returns>this</returns> + public JobRequestBuilder SetDriverConfigurationFileContents(string driverConfigurationFileContents) + { + _appParametersBuilder.SetDriverConfigurationFileContents(driverConfigurationFileContents); + return this; + } + + public JobRequestBuilder AddDriverConfigurationProviders(IEnumerable<IConfigurationProvider> configurationProviders) + { + _appParametersBuilder.AddDriverConfigurationProviders(configurationProviders); + return this; + } + } +} http://git-wip-us.apache.org/repos/asf/reef/blob/4b758fe2/lang/cs/Org.Apache.REEF.Client/API/JobSubmission.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/API/JobSubmission.cs b/lang/cs/Org.Apache.REEF.Client/API/JobSubmission.cs index a37d18a..1484d2e 100644 --- a/lang/cs/Org.Apache.REEF.Client/API/JobSubmission.cs +++ b/lang/cs/Org.Apache.REEF.Client/API/JobSubmission.cs @@ -35,26 +35,17 @@ namespace Org.Apache.REEF.Client.API private readonly string _jobIdentifier; private readonly string _driverConfigurationFileContents; - internal JobSubmission( - ISet<IConfiguration> driverConfigurations, - ISet<string> globalAssemblies, - ISet<string> globalFiles, - ISet<string> localAssemblies, - ISet<string> localFiles, - int driverMemory, - string jobIdentifier, - string driverConfigurationFileContents, - int maxAppSubmissions) + internal JobSubmission(JobRequest jobRequest) { - _driverConfigurations = driverConfigurations; - _globalAssemblies = globalAssemblies; - _globalFiles = globalFiles; - _localAssemblies = localAssemblies; - _localFiles = localFiles; - _driverMemory = driverMemory; - _jobIdentifier = jobIdentifier; - _driverConfigurationFileContents = driverConfigurationFileContents; - _maxAppSubmissions = maxAppSubmissions; + _driverConfigurations = jobRequest.DriverConfigurations; + _globalAssemblies = jobRequest.GlobalAssemblies; + _globalFiles = jobRequest.GlobalFiles; + _localAssemblies = jobRequest.LocalAssemblies; + _localFiles = jobRequest.LocalFiles; + _driverMemory = jobRequest.DriverMemory; + _jobIdentifier = jobRequest.JobIdentifier; + _driverConfigurationFileContents = jobRequest.DriverConfigurationFileContents; + _maxAppSubmissions = jobRequest.MaxApplicationSubmissions; } /// <summary> http://git-wip-us.apache.org/repos/asf/reef/blob/4b758fe2/lang/cs/Org.Apache.REEF.Client/API/JobSubmissionBuilder.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/API/JobSubmissionBuilder.cs b/lang/cs/Org.Apache.REEF.Client/API/JobSubmissionBuilder.cs index eab3fab..12a902f 100644 --- a/lang/cs/Org.Apache.REEF.Client/API/JobSubmissionBuilder.cs +++ b/lang/cs/Org.Apache.REEF.Client/API/JobSubmissionBuilder.cs @@ -17,27 +17,18 @@ using System; using System.Collections.Generic; -using System.IO; using Org.Apache.REEF.Tang.Interface; namespace Org.Apache.REEF.Client.API { + [Obsolete("Deprecated in 0.14. Will be removed.")] internal class JobSubmissionBuilder : IJobSubmissionBuilder { - private readonly ISet<IConfiguration> _driverConfigurations = new HashSet<IConfiguration>(); - private readonly ISet<string> _globalAssemblies = new HashSet<string>(); - private readonly ISet<string> _globalFiles = new HashSet<string>(); - private readonly ISet<string> _localAssemblies = new HashSet<string>(); - private readonly ISet<string> _localFiles = new HashSet<string>(); - private int _driverMemory = 512; - private int _maxAppSubmissions = 1; - private string _jobIdentifier; - private readonly ISet<IConfigurationProvider> _configurationProviders; - private string _driverConfigurationFileContents; + private readonly JobRequestBuilder _jobRequestBuilder = JobRequestBuilder.NewBuilder(); internal JobSubmissionBuilder(ISet<IConfigurationProvider> configurationProviders) { - _configurationProviders = configurationProviders; + _jobRequestBuilder.AddDriverConfigurationProviders(configurationProviders); } /// <summary> @@ -47,7 +38,7 @@ namespace Org.Apache.REEF.Client.API /// <returns></returns> public IJobSubmissionBuilder AddGlobalFile(string fileName) { - _globalFiles.Add(fileName); + _jobRequestBuilder.AddGlobalFile(fileName); return this; } @@ -58,7 +49,7 @@ namespace Org.Apache.REEF.Client.API /// <returns></returns> public IJobSubmissionBuilder AddLocalFile(string fileName) { - _localFiles.Add(fileName); + _jobRequestBuilder.AddLocalFile(fileName); return this; } @@ -69,7 +60,7 @@ namespace Org.Apache.REEF.Client.API /// <returns></returns> public IJobSubmissionBuilder AddGlobalAssembly(string fileName) { - _globalAssemblies.Add(fileName); + _jobRequestBuilder.AddGlobalAssembly(fileName); return this; } @@ -80,7 +71,7 @@ namespace Org.Apache.REEF.Client.API /// <returns></returns> public IJobSubmissionBuilder AddLocalAssembly(string fileName) { - _localAssemblies.Add(fileName); + _jobRequestBuilder.AddLocalAssembly(fileName); return this; } @@ -91,7 +82,7 @@ namespace Org.Apache.REEF.Client.API /// <returns></returns> public IJobSubmissionBuilder AddDriverConfiguration(IConfiguration configuration) { - _driverConfigurations.Add(configuration); + _jobRequestBuilder.AddDriverConfiguration(configuration); return this; } @@ -102,7 +93,7 @@ namespace Org.Apache.REEF.Client.API /// <returns></returns> public IJobSubmissionBuilder AddLocalAssemblyForType(Type type) { - AddLocalAssembly(GetAssemblyPathForType(type)); + _jobRequestBuilder.AddLocalAssemblyForType(type); return this; } @@ -113,7 +104,7 @@ namespace Org.Apache.REEF.Client.API /// <returns></returns> public IJobSubmissionBuilder AddGlobalAssemblyForType(Type type) { - AddGlobalAssembly(GetAssemblyPathForType(type)); + _jobRequestBuilder.AddGlobalAssemblyForType(type); return this; } @@ -124,7 +115,7 @@ namespace Org.Apache.REEF.Client.API /// <returns></returns> public IJobSubmissionBuilder SetJobIdentifier(string id) { - _jobIdentifier = id; + _jobRequestBuilder.SetJobIdentifier(id); return this; } @@ -135,7 +126,7 @@ namespace Org.Apache.REEF.Client.API /// <returns>this</returns> public IJobSubmissionBuilder SetDriverMemory(int driverMemoryInMb) { - _driverMemory = driverMemoryInMb; + _jobRequestBuilder.SetDriverMemory(driverMemoryInMb); return this; } @@ -144,7 +135,7 @@ namespace Org.Apache.REEF.Client.API /// </summary> public IJobSubmissionBuilder SetMaxApplicationSubmissions(int maxAppSubmissions) { - _maxAppSubmissions = maxAppSubmissions; + _jobRequestBuilder.SetMaxApplicationSubmissions(maxAppSubmissions); return this; } @@ -156,34 +147,17 @@ namespace Org.Apache.REEF.Client.API /// <returns>this</returns> public IJobSubmissionBuilder SetDriverConfigurationFileContents(string driverConfigurationFileContents) { - _driverConfigurationFileContents = driverConfigurationFileContents; + _jobRequestBuilder.SetDriverConfigurationFileContents(driverConfigurationFileContents); return this; } /// <summary> - /// Finds the path to the assembly the given Type was loaded from. - /// </summary> - /// <param name="type"></param> - /// <returns></returns> - private static string GetAssemblyPathForType(Type type) - { - var path = Uri.UnescapeDataString(new UriBuilder(type.Assembly.CodeBase).Path); - return Path.GetFullPath(path); - } - - /// <summary> /// Builds the submission /// </summary> /// <returns>IJobSubmission</returns> public IJobSubmission Build() { - foreach (var cfg in _configurationProviders) - { - _driverConfigurations.Add(cfg.GetConfiguration()); - } - - return new JobSubmission(_driverConfigurations, _globalAssemblies, _globalFiles, _localAssemblies, - _localFiles, _driverMemory, _jobIdentifier, _driverConfigurationFileContents, _maxAppSubmissions); + return new JobSubmission(_jobRequestBuilder.Build()); } } } http://git-wip-us.apache.org/repos/asf/reef/blob/4b758fe2/lang/cs/Org.Apache.REEF.Client/API/JobSubmissionBuilderFactory.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/API/JobSubmissionBuilderFactory.cs b/lang/cs/Org.Apache.REEF.Client/API/JobSubmissionBuilderFactory.cs index 1bbbf80..d7b7375 100644 --- a/lang/cs/Org.Apache.REEF.Client/API/JobSubmissionBuilderFactory.cs +++ b/lang/cs/Org.Apache.REEF.Client/API/JobSubmissionBuilderFactory.cs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +using System; using System.Collections.Generic; using Org.Apache.REEF.Common.Client.Parameters; using Org.Apache.REEF.Tang.Annotations; @@ -25,6 +26,7 @@ namespace Org.Apache.REEF.Client.API /// <summary> /// Instantiates IJobSubmissionBuilder based on configurationProviders. /// </summary> + [Obsolete("Deprecated in 0.14. Please use JobRequestBuilder instead.")] public sealed class JobSubmissionBuilderFactory { private readonly ISet<IConfigurationProvider> _configurationProviders; http://git-wip-us.apache.org/repos/asf/reef/blob/4b758fe2/lang/cs/Org.Apache.REEF.Client/Common/DriverFolderPreparationHelper.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/Common/DriverFolderPreparationHelper.cs b/lang/cs/Org.Apache.REEF.Client/Common/DriverFolderPreparationHelper.cs index a6a7ecb..7c63498 100644 --- a/lang/cs/Org.Apache.REEF.Client/Common/DriverFolderPreparationHelper.cs +++ b/lang/cs/Org.Apache.REEF.Client/Common/DriverFolderPreparationHelper.cs @@ -80,20 +80,20 @@ namespace Org.Apache.REEF.Client.Common /// <summary> /// Prepares the working directory for a Driver in driverFolderPath. /// </summary> - /// <param name="jobSubmission"></param> + /// <param name="appParameters"></param> /// <param name="driverFolderPath"></param> - internal void PrepareDriverFolder(IJobSubmission jobSubmission, string driverFolderPath) + internal void PrepareDriverFolder(AppParameters appParameters, string driverFolderPath) { Logger.Log(Level.Info, "Preparing Driver filesystem layout in " + driverFolderPath); // Setup the folder structure - CreateDefaultFolderStructure(jobSubmission, driverFolderPath); + CreateDefaultFolderStructure(appParameters, driverFolderPath); - // Add the jobSubmission into that folder structure - _fileSets.AddJobFiles(jobSubmission); + // Add the appParameters into that folder structure + _fileSets.AddJobFiles(appParameters); // Create the driver configuration - CreateDriverConfiguration(jobSubmission, driverFolderPath); + CreateDriverConfiguration(appParameters, driverFolderPath); // Add the REEF assemblies AddAssemblies(); @@ -105,15 +105,15 @@ namespace Org.Apache.REEF.Client.Common } /// <summary> - /// Merges the Configurations in jobSubmission and serializes them into the right place within driverFolderPath, + /// Merges the Configurations in appParameters and serializes them into the right place within driverFolderPath, /// assuming /// that points to a Driver's working directory. /// </summary> - /// <param name="jobSubmission"></param> + /// <param name="appParameters"></param> /// <param name="driverFolderPath"></param> - internal void CreateDriverConfiguration(IJobSubmission jobSubmission, string driverFolderPath) + internal void CreateDriverConfiguration(AppParameters appParameters, string driverFolderPath) { - var driverConfiguration = Configurations.Merge(jobSubmission.DriverConfigurations.ToArray()); + var driverConfiguration = Configurations.Merge(appParameters.DriverConfigurations.ToArray()); _configurationSerializer.ToFile(driverConfiguration, Path.Combine(driverFolderPath, _fileNames.GetClrDriverConfigurationPath())); @@ -126,9 +126,9 @@ namespace Org.Apache.REEF.Client.Common /// <summary> /// Creates the driver folder structure in this given folder as the root /// </summary> - /// <param name="jobSubmission">Job submission information</param> + /// <param name="appParameters">Job submission information</param> /// <param name="driverFolderPath">Driver folder path</param> - internal void CreateDefaultFolderStructure(IJobSubmission jobSubmission, string driverFolderPath) + internal void CreateDefaultFolderStructure(AppParameters appParameters, string driverFolderPath) { Directory.CreateDirectory(Path.Combine(driverFolderPath, _fileNames.GetReefFolderName())); Directory.CreateDirectory(Path.Combine(driverFolderPath, _fileNames.GetLocalFolderPath())); @@ -146,9 +146,9 @@ namespace Org.Apache.REEF.Client.Common } var config = DefaultDriverConfigurationFileContents; - if (!string.IsNullOrEmpty(jobSubmission.DriverConfigurationFileContents)) + if (!string.IsNullOrEmpty(appParameters.DriverConfigurationFileContents)) { - config = jobSubmission.DriverConfigurationFileContents; + config = appParameters.DriverConfigurationFileContents; } File.WriteAllText(Path.Combine(driverFolderPath, _fileNames.GetBridgeExeConfigPath()), config); } http://git-wip-us.apache.org/repos/asf/reef/blob/4b758fe2/lang/cs/Org.Apache.REEF.Client/Common/FileSets.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/Common/FileSets.cs b/lang/cs/Org.Apache.REEF.Client/Common/FileSets.cs index 35efbfd..b649502 100644 --- a/lang/cs/Org.Apache.REEF.Client/Common/FileSets.cs +++ b/lang/cs/Org.Apache.REEF.Client/Common/FileSets.cs @@ -119,7 +119,7 @@ namespace Org.Apache.REEF.Client.Common /// Adds all the files referenced in the given JobSubmission /// </summary> /// <param name="submission"></param> - internal void AddJobFiles(IJobSubmission submission) + internal void AddJobFiles(AppParameters submission) { AddToGlobalFiles(submission.GlobalFiles); AddToGlobalFiles(submission.GlobalAssemblies); http://git-wip-us.apache.org/repos/asf/reef/blob/4b758fe2/lang/cs/Org.Apache.REEF.Client/Local/LocalClient.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/Local/LocalClient.cs b/lang/cs/Org.Apache.REEF.Client/Local/LocalClient.cs index acc2a69..c6b480b 100644 --- a/lang/cs/Org.Apache.REEF.Client/Local/LocalClient.cs +++ b/lang/cs/Org.Apache.REEF.Client/Local/LocalClient.cs @@ -56,7 +56,7 @@ namespace Org.Apache.REEF.Client.Local private readonly IJavaClientLauncher _javaClientLauncher; private readonly int _maxNumberOfConcurrentEvaluators; private readonly string _runtimeFolder; - private REEFFileNames _fileNames; + private readonly REEFFileNames _fileNames; [Inject] private LocalClient(DriverFolderPreparationHelper driverFolderPreparationHelper, @@ -90,12 +90,12 @@ namespace Org.Apache.REEF.Client.Local // Intentionally left blank. } - private string CreateBootstrapAvroJobConfig(IJobSubmission jobSubmission, string driverFolder) + private string CreateBootstrapAvroJobConfig(JobParameters jobParameters, string driverFolder) { var bootstrapJobArgs = new AvroJobSubmissionParameters { jobSubmissionFolder = driverFolder, - jobId = jobSubmission.JobIdentifier, + jobId = jobParameters.JobIdentifier, }; var submissionArgsFilePath = Path.Combine(driverFolder, _fileNames.GetJobSubmissionParametersFile()); @@ -108,9 +108,9 @@ namespace Org.Apache.REEF.Client.Local return submissionArgsFilePath; } - private string CreateBootstrapAvroAppConfig(IJobSubmission jobSubmission, string driverFolder) + private string CreateBootstrapAvroAppConfig(AppParameters appParameters, string driverFolder) { - var paramInjector = TangFactory.GetTang().NewInjector(jobSubmission.DriverConfigurations.ToArray()); + var paramInjector = TangFactory.GetTang().NewInjector(appParameters.DriverConfigurations.ToArray()); var bootstrapAppArgs = new AvroAppSubmissionParameters { @@ -135,32 +135,37 @@ namespace Org.Apache.REEF.Client.Local return submissionArgsFilePath; } - private string PrepareDriverFolder(IJobSubmission jobSubmission) + private string PrepareDriverFolder(JobRequest jobRequest) { // Prepare the job submission folder - var jobFolder = CreateJobFolder(jobSubmission.JobIdentifier); + var jobFolder = CreateJobFolder(jobRequest.JobIdentifier); var driverFolder = Path.Combine(jobFolder, DriverFolderName); Logger.Log(Level.Info, "Preparing driver folder in " + driverFolder); - _driverFolderPreparationHelper.PrepareDriverFolder(jobSubmission, driverFolder); + _driverFolderPreparationHelper.PrepareDriverFolder(jobRequest.AppParameters, driverFolder); return driverFolder; } public void Submit(IJobSubmission jobSubmission) { - var driverFolder = PrepareDriverFolder(jobSubmission); - var submissionJobArgsFilePath = CreateBootstrapAvroJobConfig(jobSubmission, driverFolder); - var submissionAppArgsFilePath = CreateBootstrapAvroAppConfig(jobSubmission, driverFolder); + Submit(JobRequest.FromJobSubmission(jobSubmission)); + } + + public void Submit(JobRequest jobRequest) + { + var driverFolder = PrepareDriverFolder(jobRequest); + var submissionJobArgsFilePath = CreateBootstrapAvroJobConfig(jobRequest.JobParameters, driverFolder); + var submissionAppArgsFilePath = CreateBootstrapAvroAppConfig(jobRequest.AppParameters, driverFolder); _javaClientLauncher.Launch(JavaClassName, submissionJobArgsFilePath, submissionAppArgsFilePath); Logger.Log(Level.Info, "Submitted the Driver for execution."); } - public IJobSubmissionResult SubmitAndGetJobStatus(IJobSubmission jobSubmission) + public IJobSubmissionResult SubmitAndGetJobStatus(JobRequest jobRequest) { - var driverFolder = PrepareDriverFolder(jobSubmission); - var submissionJobArgsFilePath = CreateBootstrapAvroJobConfig(jobSubmission, driverFolder); - var submissionAppArgsFilePath = CreateBootstrapAvroAppConfig(jobSubmission, driverFolder); + var driverFolder = PrepareDriverFolder(jobRequest); + var submissionJobArgsFilePath = CreateBootstrapAvroJobConfig(jobRequest.JobParameters, driverFolder); + var submissionAppArgsFilePath = CreateBootstrapAvroAppConfig(jobRequest.AppParameters, driverFolder); Task.Run(() => _javaClientLauncher.Launch(JavaClassName, submissionJobArgsFilePath, submissionAppArgsFilePath)); @@ -169,7 +174,7 @@ namespace Org.Apache.REEF.Client.Local var msg = string.Format(CultureInfo.CurrentCulture, "Submitted the Driver for execution. Returned driverUrl is: {0}.", result.DriverUrl); - Logger.Log(Level.Info, msg); + Logger.Log(Level.Info, msg); return result; } http://git-wip-us.apache.org/repos/asf/reef/blob/4b758fe2/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj b/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj index 608035b..babf263 100644 --- a/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj +++ b/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj @@ -59,12 +59,18 @@ under the License. <Reference Include="System.Runtime.Serialization" /> </ItemGroup> <ItemGroup> + <Compile Include="API\AppParameters.cs" /> + <Compile Include="API\AppParametersBuilder.cs" /> <Compile Include="API\ClientFactory.cs" /> <Compile Include="API\Exceptions\ClasspathException.cs" /> <Compile Include="API\Exceptions\JavaNotFoundException.cs" /> <Compile Include="API\IJobSubmission.cs" /> <Compile Include="API\IJobSubmissionBuilder.cs" /> <Compile Include="API\IREEFClient.cs" /> + <Compile Include="API\JobParameters.cs" /> + <Compile Include="API\JobParametersBuilder.cs" /> + <Compile Include="API\JobRequest.cs" /> + <Compile Include="API\JobRequestBuilder.cs" /> <Compile Include="API\JobSubmission.cs" /> <Compile Include="API\JobSubmissionBuilder.cs" /> <Compile Include="API\JobSubmissionBuilderFactory.cs" /> http://git-wip-us.apache.org/repos/asf/reef/blob/4b758fe2/lang/cs/Org.Apache.REEF.Client/YARN/YARNREEFClient.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/YARNREEFClient.cs b/lang/cs/Org.Apache.REEF.Client/YARN/YARNREEFClient.cs index 174ae2c..cbf7eb2 100644 --- a/lang/cs/Org.Apache.REEF.Client/YARN/YARNREEFClient.cs +++ b/lang/cs/Org.Apache.REEF.Client/YARN/YARNREEFClient.cs @@ -26,7 +26,6 @@ using Org.Apache.REEF.Client.Yarn.RestClient; using Org.Apache.REEF.Client.YARN; using Org.Apache.REEF.Client.YARN.RestClient.DataModel; using Org.Apache.REEF.Common.Files; -using Org.Apache.REEF.Driver.Bridge; using Org.Apache.REEF.Tang.Annotations; using Org.Apache.REEF.Tang.Implementations.Tang; using Org.Apache.REEF.Utilities.Logging; @@ -65,20 +64,25 @@ namespace Org.Apache.REEF.Client.Yarn public void Submit(IJobSubmission jobSubmission) { + Submit(JobRequest.FromJobSubmission(jobSubmission)); + } + + public void Submit(JobRequest jobRequest) + { // Prepare the job submission folder - var driverFolderPath = CreateDriverFolder(jobSubmission.JobIdentifier); + var driverFolderPath = CreateDriverFolder(jobRequest.JobIdentifier); Logger.Log(Level.Info, "Preparing driver folder in " + driverFolderPath); - Launch(jobSubmission, driverFolderPath); + Launch(jobRequest, driverFolderPath); } - public IJobSubmissionResult SubmitAndGetJobStatus(IJobSubmission jobSubmission) + public IJobSubmissionResult SubmitAndGetJobStatus(JobRequest jobRequest) { // Prepare the job submission folder - var driverFolderPath = CreateDriverFolder(jobSubmission.JobIdentifier); + var driverFolderPath = CreateDriverFolder(jobRequest.JobIdentifier); Logger.Log(Level.Info, "Preparing driver folder in " + driverFolderPath); - Launch(jobSubmission, driverFolderPath); + Launch(jobRequest, driverFolderPath); var pointerFileName = Path.Combine(driverFolderPath, _fileNames.DriverHttpEndpoint); var jobSubmitionResultImpl = new YarnJobSubmissionResult(this, pointerFileName); @@ -91,6 +95,11 @@ namespace Org.Apache.REEF.Client.Yarn return jobSubmitionResultImpl; } + public IJobSubmissionResult SubmitAndGetJobStatus(IJobSubmission jobSubmission) + { + return SubmitAndGetJobStatus(JobRequest.FromJobSubmission(jobSubmission)); + } + /// <summary> /// Pull Job status from Yarn for the given appId /// </summary> @@ -106,18 +115,18 @@ namespace Org.Apache.REEF.Client.Yarn return application.FinalStatus; } - private void Launch(IJobSubmission jobSubmission, string driverFolderPath) + private void Launch(JobRequest jobRequest, string driverFolderPath) { - _driverFolderPreparationHelper.PrepareDriverFolder(jobSubmission, driverFolderPath); + _driverFolderPreparationHelper.PrepareDriverFolder(jobRequest.AppParameters, driverFolderPath); // TODO: Remove this when we have a generalized way to pass config to java - var paramInjector = TangFactory.GetTang().NewInjector(jobSubmission.DriverConfigurations.ToArray()); - var submissionJobArgsFilePath = _paramSerializer.SerializeJobFile(jobSubmission, paramInjector, driverFolderPath); - var submissionAppArgsFilePath = _paramSerializer.SerializeAppFile(jobSubmission, paramInjector, driverFolderPath); + var paramInjector = TangFactory.GetTang().NewInjector(jobRequest.DriverConfigurations.ToArray()); + var submissionJobArgsFilePath = _paramSerializer.SerializeJobFile(jobRequest.JobParameters, paramInjector, driverFolderPath); + var submissionAppArgsFilePath = _paramSerializer.SerializeAppFile(jobRequest.AppParameters, paramInjector, driverFolderPath); // Submit the driver _javaClientLauncher.Launch(JavaClassName, submissionJobArgsFilePath, submissionAppArgsFilePath); - Logger.Log(Level.Info, "Submitted the Driver for execution." + jobSubmission.JobIdentifier); + Logger.Log(Level.Info, "Submitted the Driver for execution." + jobRequest.JobIdentifier); } /// <summary> http://git-wip-us.apache.org/repos/asf/reef/blob/4b758fe2/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFDotNetClient.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFDotNetClient.cs b/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFDotNetClient.cs index 41697f2..ae109c1 100644 --- a/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFDotNetClient.cs +++ b/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFDotNetClient.cs @@ -26,7 +26,6 @@ using Org.Apache.REEF.Client.Yarn; using Org.Apache.REEF.Client.Yarn.RestClient; using Org.Apache.REEF.Client.YARN.RestClient.DataModel; using Org.Apache.REEF.Common.Files; -using Org.Apache.REEF.Driver.Bridge; using Org.Apache.REEF.Tang.Annotations; using Org.Apache.REEF.Tang.Implementations.Tang; using Org.Apache.REEF.Utilities.Attributes; @@ -71,9 +70,9 @@ namespace Org.Apache.REEF.Client.YARN _paramSerializer = paramSerializer; } - public void Submit(IJobSubmission jobSubmission) + public void Submit(JobRequest jobRequest) { - string jobId = jobSubmission.JobIdentifier; + string jobId = jobRequest.JobIdentifier; // todo: Future client interface should be async. // Using GetAwaiter().GetResult() instead of .Result to avoid exception @@ -90,13 +89,13 @@ namespace Org.Apache.REEF.Client.YARN try { Log.Log(Level.Verbose, "Preparing driver folder in {0}", localDriverFolderPath); - _driverFolderPreparationHelper.PrepareDriverFolder(jobSubmission, localDriverFolderPath); + _driverFolderPreparationHelper.PrepareDriverFolder(jobRequest.AppParameters, localDriverFolderPath); // prepare configuration - var paramInjector = TangFactory.GetTang().NewInjector(jobSubmission.DriverConfigurations.ToArray()); + var paramInjector = TangFactory.GetTang().NewInjector(jobRequest.DriverConfigurations.ToArray()); - _paramSerializer.SerializeAppFile(jobSubmission, paramInjector, localDriverFolderPath); - _paramSerializer.SerializeJobFile(jobSubmission, localDriverFolderPath, jobSubmissionDirectory); + _paramSerializer.SerializeAppFile(jobRequest.AppParameters, paramInjector, localDriverFolderPath); + _paramSerializer.SerializeJobFile(jobRequest.JobParameters, localDriverFolderPath, jobSubmissionDirectory); var archiveResource = _jobResourceUploader.UploadArchiveResource(localDriverFolderPath, jobSubmissionDirectory); @@ -111,10 +110,12 @@ namespace Org.Apache.REEF.Client.YARN // submit job Log.Log(Level.Verbose, @"Assigned application id {0}", applicationId); - var submissionReq = CreateApplicationSubmissionRequest(jobSubmission, + var submissionReq = CreateApplicationSubmissionRequest( + jobRequest.JobParameters, applicationId, - jobSubmission.MaxApplicationSubmissions, + jobRequest.MaxApplicationSubmissions, jobResources); + var submittedApplication = _yarnRMClient.SubmitApplicationAsync(submissionReq).GetAwaiter().GetResult(); Log.Log(Level.Info, @"Submitted application {0}", submittedApplication.Id); } @@ -127,11 +128,17 @@ namespace Org.Apache.REEF.Client.YARN } } - public IJobSubmissionResult SubmitAndGetJobStatus(IJobSubmission jobSubmission) + public IJobSubmissionResult SubmitAndGetJobStatus(JobRequest jobRequest) { throw new NotSupportedException(); } + [Obsolete("Deprecated in 0.14, please use Submit(JobRequest)")] + public void Submit(IJobSubmission jobSubmission) + { + Submit(JobRequest.FromJobSubmission(jobSubmission)); + } + public async Task<FinalState> GetJobFinalStatus(string appId) { var application = await _yarnRMClient.GetApplicationAsync(appId).ConfigureAwait(false); @@ -139,7 +146,7 @@ namespace Org.Apache.REEF.Client.YARN } private SubmitApplication CreateApplicationSubmissionRequest( - IJobSubmission jobSubmission, + JobParameters jobParameters, string appId, int maxApplicationSubmissions, IReadOnlyCollection<JobResource> jobResources) @@ -156,10 +163,10 @@ namespace Org.Apache.REEF.Client.YARN var submitApplication = new SubmitApplication { ApplicationId = appId, - ApplicationName = jobSubmission.JobIdentifier, + ApplicationName = jobParameters.JobIdentifier, AmResource = new Resouce { - MemoryMB = jobSubmission.DriverMemory, + MemoryMB = jobParameters.DriverMemoryInMB, VCores = 1 // keeping parity with existing code }, MaxAppAttempts = maxApplicationSubmissions, http://git-wip-us.apache.org/repos/asf/reef/blob/4b758fe2/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFDotNetParamSerializer.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFDotNetParamSerializer.cs b/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFDotNetParamSerializer.cs index 91445df..5981c13 100644 --- a/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFDotNetParamSerializer.cs +++ b/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFDotNetParamSerializer.cs @@ -44,9 +44,9 @@ namespace Org.Apache.REEF.Client.YARN /// <summary> /// Serializes the application parameters to reef/local/app-submission-params.json. /// </summary> - internal void SerializeAppFile(IJobSubmission jobSubmission, IInjector paramInjector, string localDriverFolderPath) + internal void SerializeAppFile(AppParameters appParameters, IInjector paramInjector, string localDriverFolderPath) { - var serializedArgs = SerializeAppArgsToBytes(jobSubmission, paramInjector, localDriverFolderPath); + var serializedArgs = SerializeAppArgsToBytes(appParameters, paramInjector, localDriverFolderPath); var submissionAppArgsFilePath = Path.Combine( localDriverFolderPath, _fileNames.GetLocalFolderPath(), _fileNames.GetAppSubmissionParametersFile()); @@ -57,7 +57,7 @@ namespace Org.Apache.REEF.Client.YARN } } - internal byte[] SerializeAppArgsToBytes(IJobSubmission jobSubmission, IInjector paramInjector, string localDriverFolderPath) + internal byte[] SerializeAppArgsToBytes(AppParameters appParameters, IInjector paramInjector, string localDriverFolderPath) { var avroAppSubmissionParameters = new AvroAppSubmissionParameters { @@ -79,9 +79,9 @@ namespace Org.Apache.REEF.Client.YARN /// <summary> /// Serializes the job parameters to job-submission-params.json. /// </summary> - internal void SerializeJobFile(IJobSubmission jobSubmission, string localDriverFolderPath, string jobSubmissionDirectory) + internal void SerializeJobFile(JobParameters jobParameters, string localDriverFolderPath, string jobSubmissionDirectory) { - var serializedArgs = SerializeJobArgsToBytes(jobSubmission, localDriverFolderPath, jobSubmissionDirectory); + var serializedArgs = SerializeJobArgsToBytes(jobParameters, localDriverFolderPath, jobSubmissionDirectory); var submissionJobArgsFilePath = Path.Combine(localDriverFolderPath, _fileNames.GetJobSubmissionParametersFile()); @@ -92,11 +92,11 @@ namespace Org.Apache.REEF.Client.YARN } } - internal byte[] SerializeJobArgsToBytes(IJobSubmission jobSubmission, string localDriverFolderPath, string jobSubmissionDirectory) + internal byte[] SerializeJobArgsToBytes(JobParameters jobParameters, string localDriverFolderPath, string jobSubmissionDirectory) { var avroJobSubmissionParameters = new AvroJobSubmissionParameters { - jobId = jobSubmission.JobIdentifier, + jobId = jobParameters.JobIdentifier, jobSubmissionFolder = localDriverFolderPath }; http://git-wip-us.apache.org/repos/asf/reef/blob/4b758fe2/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFParamSerializer.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFParamSerializer.cs b/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFParamSerializer.cs index 9bfa93c..cb207a4 100644 --- a/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFParamSerializer.cs +++ b/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFParamSerializer.cs @@ -56,9 +56,9 @@ namespace Org.Apache.REEF.Client.YARN /// <summary> /// Serializes the application parameters to reef/local/app-submission-params.json. /// </summary> - internal string SerializeAppFile(IJobSubmission jobSubmission, IInjector paramInjector, string driverFolderPath) + internal string SerializeAppFile(AppParameters appParameters, IInjector paramInjector, string driverFolderPath) { - var serializedArgs = SerializeAppArgsToBytes(jobSubmission, paramInjector); + var serializedArgs = SerializeAppArgsToBytes(appParameters, paramInjector); var submissionArgsFilePath = Path.Combine(driverFolderPath, _fileNames.GetAppSubmissionParametersFile()); using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew)) @@ -69,7 +69,7 @@ namespace Org.Apache.REEF.Client.YARN return submissionArgsFilePath; } - internal byte[] SerializeAppArgsToBytes(IJobSubmission jobSubmission, IInjector paramInjector) + internal byte[] SerializeAppArgsToBytes(AppParameters appParameters, IInjector paramInjector) { var avroAppSubmissionParameters = new AvroAppSubmissionParameters { @@ -90,9 +90,9 @@ namespace Org.Apache.REEF.Client.YARN /// <summary> /// Serializes the job parameters to job-submission-params.json. /// </summary> - internal string SerializeJobFile(IJobSubmission jobSubmission, IInjector paramInjector, string driverFolderPath) + internal string SerializeJobFile(JobParameters jobParameters, IInjector paramInjector, string driverFolderPath) { - var serializedArgs = SerializeJobArgsToBytes(jobSubmission, paramInjector, driverFolderPath); + var serializedArgs = SerializeJobArgsToBytes(jobParameters, paramInjector, driverFolderPath); var submissionArgsFilePath = Path.Combine(driverFolderPath, _fileNames.GetJobSubmissionParametersFile()); using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew)) @@ -103,11 +103,11 @@ namespace Org.Apache.REEF.Client.YARN return submissionArgsFilePath; } - internal byte[] SerializeJobArgsToBytes(IJobSubmission jobSubmission, IInjector paramInjector, string driverFolderPath) + internal byte[] SerializeJobArgsToBytes(JobParameters jobParameters, IInjector paramInjector, string driverFolderPath) { var avroJobSubmissionParameters = new AvroJobSubmissionParameters { - jobId = jobSubmission.JobIdentifier, + jobId = jobParameters.JobIdentifier, jobSubmissionFolder = driverFolderPath }; @@ -117,16 +117,16 @@ namespace Org.Apache.REEF.Client.YARN sharedJobSubmissionParameters = avroJobSubmissionParameters }; - var maxApplicationSubmissions = jobSubmission.MaxApplicationSubmissions == 1 + var maxApplicationSubmissions = jobParameters.MaxApplicationSubmissions == 1 ? paramInjector.GetNamedInstance<DriverBridgeConfigurationOptions.MaxApplicationSubmissions, int>() - : jobSubmission.MaxApplicationSubmissions; + : jobParameters.MaxApplicationSubmissions; var avroYarnClusterJobSubmissionParameters = new AvroYarnClusterJobSubmissionParameters { securityTokenKind = _securityTokenKind, securityTokenService = _securityTokenService, yarnJobSubmissionParameters = avroYarnJobSubmissionParameters, - driverMemory = jobSubmission.DriverMemory, + driverMemory = jobParameters.DriverMemoryInMB, maxApplicationSubmissions = maxApplicationSubmissions }; http://git-wip-us.apache.org/repos/asf/reef/blob/4b758fe2/lang/cs/Org.Apache.REEF.Examples.AllHandlers/AllHandlers.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Examples.AllHandlers/AllHandlers.cs b/lang/cs/Org.Apache.REEF.Examples.AllHandlers/AllHandlers.cs index 9f8aae3..3d3eb0a 100644 --- a/lang/cs/Org.Apache.REEF.Examples.AllHandlers/AllHandlers.cs +++ b/lang/cs/Org.Apache.REEF.Examples.AllHandlers/AllHandlers.cs @@ -40,13 +40,13 @@ namespace Org.Apache.REEF.Examples.AllHandlers private const string Local = "local"; private const string YARN = "yarn"; private readonly IREEFClient _reefClient; - private readonly JobSubmissionBuilderFactory _jobSubmissionBuilderFactory; + private readonly JobRequestBuilder _jobRequestBuilder; [Inject] - private AllHandlers(IREEFClient reefClient, JobSubmissionBuilderFactory jobSubmissionBuilderFactory) + private AllHandlers(IREEFClient reefClient, JobRequestBuilder jobRequestBuilder) { _reefClient = reefClient; - _jobSubmissionBuilderFactory = jobSubmissionBuilderFactory; + _jobRequestBuilder = jobRequestBuilder; } private IJobSubmissionResult Run() @@ -76,7 +76,7 @@ namespace Org.Apache.REEF.Examples.AllHandlers .BindSetEntry<DriverBridgeConfigurationOptions.SetOfAssemblies, string>(typeof(NameClient).Assembly.GetName().Name) .Build(); - var helloJobSubmission = _jobSubmissionBuilderFactory.GetJobSubmissionBuilder() + var helloJobSubmission = _jobRequestBuilder .AddDriverConfiguration(driverConfig) .AddGlobalAssemblyForType(typeof(HelloDriverStartHandler)) .SetJobIdentifier("HelloDriver") http://git-wip-us.apache.org/repos/asf/reef/blob/4b758fe2/lang/cs/Org.Apache.REEF.Examples.DriverRestart/DriverRestart.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Examples.DriverRestart/DriverRestart.cs b/lang/cs/Org.Apache.REEF.Examples.DriverRestart/DriverRestart.cs index d481489..7b0bdb6 100644 --- a/lang/cs/Org.Apache.REEF.Examples.DriverRestart/DriverRestart.cs +++ b/lang/cs/Org.Apache.REEF.Examples.DriverRestart/DriverRestart.cs @@ -34,13 +34,13 @@ namespace Org.Apache.REEF.Examples.DriverRestart public sealed class DriverRestart { private readonly IREEFClient _reefClient; - private readonly JobSubmissionBuilderFactory _jobSubmissionBuilderFactory; + private readonly JobRequestBuilder _jobRequestBuilder; [Inject] - private DriverRestart(IREEFClient reefClient, JobSubmissionBuilderFactory jobSubmissionBuilderFactory) + private DriverRestart(IREEFClient reefClient, JobRequestBuilder jobRequestBuilder) { _reefClient = reefClient; - _jobSubmissionBuilderFactory = jobSubmissionBuilderFactory; + _jobRequestBuilder = jobRequestBuilder; } /// <summary> @@ -67,7 +67,7 @@ namespace Org.Apache.REEF.Examples.DriverRestart .Build(); // The JobSubmission contains the Driver configuration as well as the files needed on the Driver. - var restartJobSubmission = _jobSubmissionBuilderFactory.GetJobSubmissionBuilder() + var restartJobSubmission = _jobRequestBuilder .AddDriverConfiguration(driverConfiguration) .AddGlobalAssemblyForType(typeof(HelloRestartDriver)) .SetJobIdentifier("DriverRestart_" + Guid.NewGuid().ToString().Substring(0, 6)) http://git-wip-us.apache.org/repos/asf/reef/blob/4b758fe2/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs index 0a3ed65..8c63953 100644 --- a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs +++ b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs @@ -38,13 +38,13 @@ namespace Org.Apache.REEF.Examples.HelloREEF private const string YARN = "yarn"; private const string YARNRest = "yarnrest"; private readonly IREEFClient _reefClient; - private readonly JobSubmissionBuilderFactory _jobSubmissionBuilderFactory; + private readonly JobRequestBuilder _jobRequestBuilder; [Inject] - private HelloREEF(IREEFClient reefClient, JobSubmissionBuilderFactory jobSubmissionBuilderFactory) + private HelloREEF(IREEFClient reefClient, JobRequestBuilder jobRequestBuilder) { _reefClient = reefClient; - _jobSubmissionBuilderFactory = jobSubmissionBuilderFactory; + _jobRequestBuilder = jobRequestBuilder; } /// <summary> @@ -59,13 +59,13 @@ namespace Org.Apache.REEF.Examples.HelloREEF .Build(); // The JobSubmission contains the Driver configuration as well as the files needed on the Driver. - var helloJobSubmission = _jobSubmissionBuilderFactory.GetJobSubmissionBuilder() + var helloJobRequest = _jobRequestBuilder .AddDriverConfiguration(helloDriverConfiguration) .AddGlobalAssemblyForType(typeof(HelloDriver)) .SetJobIdentifier("HelloREEF") .Build(); - _reefClient.Submit(helloJobSubmission); + _reefClient.Submit(helloJobRequest); } /// <summary> http://git-wip-us.apache.org/repos/asf/reef/blob/4b758fe2/lang/cs/Org.Apache.REEF.IMRU/OnREEF/Client/REEFIMRUClient.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.IMRU/OnREEF/Client/REEFIMRUClient.cs b/lang/cs/Org.Apache.REEF.IMRU/OnREEF/Client/REEFIMRUClient.cs index 672df9a..c2f82fd 100644 --- a/lang/cs/Org.Apache.REEF.IMRU/OnREEF/Client/REEFIMRUClient.cs +++ b/lang/cs/Org.Apache.REEF.IMRU/OnREEF/Client/REEFIMRUClient.cs @@ -48,17 +48,19 @@ namespace Org.Apache.REEF.IMRU.OnREEF.Client private static readonly Logger Logger = Logger.GetLogger(typeof(REEFIMRUClient)); private readonly IREEFClient _reefClient; - private readonly JobSubmissionBuilderFactory _jobSubmissionBuilderFactory; + private readonly JobRequestBuilder _jobRequestBuilder; private readonly AvroConfigurationSerializer _configurationSerializer; private IJobSubmissionResult _jobSubmissionResult; [Inject] - private REEFIMRUClient(IREEFClient reefClient, AvroConfigurationSerializer configurationSerializer, - JobSubmissionBuilderFactory jobSubmissionBuilderFactory) + private REEFIMRUClient( + IREEFClient reefClient, + AvroConfigurationSerializer configurationSerializer, + JobRequestBuilder jobRequestBuilder) { _reefClient = reefClient; _configurationSerializer = configurationSerializer; - _jobSubmissionBuilderFactory = jobSubmissionBuilderFactory; + _jobRequestBuilder = jobRequestBuilder; } /// <summary> @@ -142,7 +144,7 @@ namespace Org.Apache.REEF.IMRU.OnREEF.Client .Build(); // The JobSubmission contains the Driver configuration as well as the files needed on the Driver. - var imruJobSubmission = _jobSubmissionBuilderFactory.GetJobSubmissionBuilder() + var imruJobSubmission = _jobRequestBuilder .AddDriverConfiguration(imruDriverConfiguration) .AddGlobalAssemblyForType(typeof(IMRUDriver<TMapInput, TMapOutput, TResult>)) .SetJobIdentifier(jobDefinition.JobName) http://git-wip-us.apache.org/repos/asf/reef/blob/4b758fe2/lang/cs/Org.Apache.REEF.Network.Examples.Client/BroadcastAndReduceClient.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Network.Examples.Client/BroadcastAndReduceClient.cs b/lang/cs/Org.Apache.REEF.Network.Examples.Client/BroadcastAndReduceClient.cs index d962d16..f2871cc 100644 --- a/lang/cs/Org.Apache.REEF.Network.Examples.Client/BroadcastAndReduceClient.cs +++ b/lang/cs/Org.Apache.REEF.Network.Examples.Client/BroadcastAndReduceClient.cs @@ -96,8 +96,8 @@ namespace Org.Apache.REEF.Network.Examples.Client { IInjector injector = TangFactory.GetTang().NewInjector(GetRuntimeConfiguration(runOnYarn, numberOfEvaluator, runtimeFolder)); var reefClient = injector.GetInstance<IREEFClient>(); - var jobSubmissionBuilderFactory = injector.GetInstance<JobSubmissionBuilderFactory>(); - var jobSubmission = jobSubmissionBuilderFactory.GetJobSubmissionBuilder() + var jobRequestBuilder = injector.GetInstance<JobRequestBuilder>(); + var jobSubmission = jobRequestBuilder .AddDriverConfiguration(driverConfig) .AddGlobalAssemblyForType(globalAssemblyType) .SetJobIdentifier(jobIdentifier)
