Repository: incubator-reef Updated Branches: refs/heads/master a6e2218b5 -> 06db1ab1f
[REEF-840] IIMRUClient does not need to have generic type This addressed the issue by * removing generics from the IIMRU Client class definition and shifting it to Submit function JIRA: [REEF-840](https://issues.apache.org/jira/browse/REEF-840) This closes #562 Project: http://git-wip-us.apache.org/repos/asf/incubator-reef/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-reef/commit/06db1ab1 Tree: http://git-wip-us.apache.org/repos/asf/incubator-reef/tree/06db1ab1 Diff: http://git-wip-us.apache.org/repos/asf/incubator-reef/diff/06db1ab1 Branch: refs/heads/master Commit: 06db1ab1fef6dd87e2eaa99b4c1b228c896a07ec Parents: a6e2218 Author: Dhruv <dh...@apache.org> Authored: Mon Oct 12 22:27:09 2015 -0700 Committer: Julia Wang <juw...@microsoft.com> Committed: Tue Oct 13 00:12:43 2015 -0700 ---------------------------------------------------------------------- .../MapperCount/MapperCount.cs | 6 +++--- .../OnREEFIMRURunTimeConfiguration.cs | 4 ++-- .../PipelinedBroadcastAndReduce.cs | 6 +++--- .../MapperCountTest.cs | 4 ++-- lang/cs/Org.Apache.REEF.IMRU/API/IIMRUClient.cs | 18 ++++++++---------- .../InProcess/InProcessIMRUClient.cs | 19 ++++++++++--------- .../InProcess/InProcessIMRUConfiguration.cs | 11 ++++------- .../OnREEF/Client/REEFIMRUClient.cs | 14 +++++++------- .../OnREEF/Client/REEFIMRUClientConfiguration.cs | 8 ++++---- 9 files changed, 43 insertions(+), 47 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/06db1ab1/lang/cs/Org.Apache.REEF.IMRU.Examples/MapperCount/MapperCount.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.IMRU.Examples/MapperCount/MapperCount.cs b/lang/cs/Org.Apache.REEF.IMRU.Examples/MapperCount/MapperCount.cs index 76b2e37..9fed04b 100644 --- a/lang/cs/Org.Apache.REEF.IMRU.Examples/MapperCount/MapperCount.cs +++ b/lang/cs/Org.Apache.REEF.IMRU.Examples/MapperCount/MapperCount.cs @@ -31,10 +31,10 @@ namespace Org.Apache.REEF.IMRU.Examples.MapperCount /// </summary> public sealed class MapperCount { - private readonly IIMRUClient<int, int, int> _imruClient; + private readonly IIMRUClient _imruClient; [Inject] - private MapperCount(IIMRUClient<int, int, int> imruClient) + private MapperCount(IIMRUClient imruClient) { _imruClient = imruClient; } @@ -45,7 +45,7 @@ namespace Org.Apache.REEF.IMRU.Examples.MapperCount /// <returns>The number of MapFunction instances that are part of the job.</returns> public int Run(int numberofMappers) { - var results = _imruClient.Submit( + var results = _imruClient.Submit<int, int, int>( new IMRUJobDefinitionBuilder() .SetMapFunctionConfiguration(IMRUMapConfiguration<int, int>.ConfigurationModule .Set(IMRUMapConfiguration<int, int>.MapFunction, GenericType<IdentityMapFunction>.Class) http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/06db1ab1/lang/cs/Org.Apache.REEF.IMRU.Examples/OnREEFIMRURunTimeConfiguration.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.IMRU.Examples/OnREEFIMRURunTimeConfiguration.cs b/lang/cs/Org.Apache.REEF.IMRU.Examples/OnREEFIMRURunTimeConfiguration.cs index eaf99b2..c13efa0 100644 --- a/lang/cs/Org.Apache.REEF.IMRU.Examples/OnREEFIMRURunTimeConfiguration.cs +++ b/lang/cs/Org.Apache.REEF.IMRU.Examples/OnREEFIMRURunTimeConfiguration.cs @@ -43,7 +43,7 @@ namespace Org.Apache.REEF.IMRU.Examples { IConfiguration runtimeConfig; IConfiguration imruClientConfig = - REEFIMRUClientConfiguration<TMapInput, TMapOutput, TResult>.ConfigurationModule.Build(); + REEFIMRUClientConfiguration.ConfigurationModule.Build(); if (runTimeDir.Length != 0) { @@ -71,7 +71,7 @@ namespace Org.Apache.REEF.IMRU.Examples internal static IConfiguration GetYarnIMRUConfiguration() { IConfiguration imruClientConfig = - REEFIMRUClientConfiguration<TMapInput, TMapOutput, TResult>.ConfigurationModule.Build(); + REEFIMRUClientConfiguration.ConfigurationModule.Build(); var fileSystemConfig = HadoopFileSystemConfiguration.ConfigurationModule.Build(); http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/06db1ab1/lang/cs/Org.Apache.REEF.IMRU.Examples/PipelinedBroadcastReduce/PipelinedBroadcastAndReduce.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.IMRU.Examples/PipelinedBroadcastReduce/PipelinedBroadcastAndReduce.cs b/lang/cs/Org.Apache.REEF.IMRU.Examples/PipelinedBroadcastReduce/PipelinedBroadcastAndReduce.cs index 8f1cbc7..b642e37 100644 --- a/lang/cs/Org.Apache.REEF.IMRU.Examples/PipelinedBroadcastReduce/PipelinedBroadcastAndReduce.cs +++ b/lang/cs/Org.Apache.REEF.IMRU.Examples/PipelinedBroadcastReduce/PipelinedBroadcastAndReduce.cs @@ -32,10 +32,10 @@ namespace Org.Apache.REEF.IMRU.Examples.PipelinedBroadcastReduce /// </summary> public sealed class PipelinedBroadcastAndReduce { - private readonly IIMRUClient<int[], int[], int[]> _imruClient; + private readonly IIMRUClient _imruClient; [Inject] - private PipelinedBroadcastAndReduce(IIMRUClient<int[], int[], int[]> imruClient) + private PipelinedBroadcastAndReduce(IIMRUClient imruClient) { _imruClient = imruClient; } @@ -75,7 +75,7 @@ namespace Org.Apache.REEF.IMRU.Examples.PipelinedBroadcastReduce chunkSize.ToString(CultureInfo.InvariantCulture)) .Build(); - var results = _imruClient.Submit( + var results = _imruClient.Submit<int[], int[], int[]>( new IMRUJobDefinitionBuilder() .SetMapFunctionConfiguration(IMRUMapConfiguration<int[], int[]>.ConfigurationModule .Set(IMRUMapConfiguration<int[], int[]>.MapFunction, http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/06db1ab1/lang/cs/Org.Apache.REEF.IMRU.Tests/MapperCountTest.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.IMRU.Tests/MapperCountTest.cs b/lang/cs/Org.Apache.REEF.IMRU.Tests/MapperCountTest.cs index ad0e477..2874e13 100644 --- a/lang/cs/Org.Apache.REEF.IMRU.Tests/MapperCountTest.cs +++ b/lang/cs/Org.Apache.REEF.IMRU.Tests/MapperCountTest.cs @@ -41,8 +41,8 @@ namespace Org.Apache.REEF.IMRU.Tests var tested = TangFactory.GetTang() .NewInjector( - InProcessIMRUConfiguration<int, int, int>.ConfigurationModule - .Set(InProcessIMRUConfiguration<int, int, int>.NumberOfMappers, NumberOfMappers.ToString()) + InProcessIMRUConfiguration.ConfigurationModule + .Set(InProcessIMRUConfiguration.NumberOfMappers, NumberOfMappers.ToString()) .Build() ) .GetInstance<MapperCount>(); http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/06db1ab1/lang/cs/Org.Apache.REEF.IMRU/API/IIMRUClient.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.IMRU/API/IIMRUClient.cs b/lang/cs/Org.Apache.REEF.IMRU/API/IIMRUClient.cs index 247d721..51682de 100644 --- a/lang/cs/Org.Apache.REEF.IMRU/API/IIMRUClient.cs +++ b/lang/cs/Org.Apache.REEF.IMRU/API/IIMRUClient.cs @@ -21,18 +21,16 @@ using System.Collections.Generic; namespace Org.Apache.REEF.IMRU.API { - /// <summary> - /// Job submission interface for IMRU jobs. - /// </summary> - /// <typeparam name="TMapInput">The type of the side information provided to the Map function</typeparam> - /// <typeparam name="TMapOutput">The return type of the Map function</typeparam> - /// <typeparam name="TResult">The return type of the computation.</typeparam> - public interface IIMRUClient<TMapInput, TMapOutput, out TResult> - { + public interface IIMRUClient + { /// <summary> /// Submit the given job for execution. /// </summary> - /// <param name="jobDefinition"></param> - IEnumerable<TResult> Submit(IMRUJobDefinition jobDefinition); + /// <typeparam name="TMapInput">The type of the side information provided to the Map function</typeparam> + /// <typeparam name="TMapOutput">The return type of the Map function</typeparam> + /// <typeparam name="TResult">The return type of the computation.</typeparam> + /// <param name="jobDefinition">IMRU job definition</param> + /// <returns>Result of IMRU</returns> + IEnumerable<TResult> Submit<TMapInput, TMapOutput, TResult>(IMRUJobDefinition jobDefinition); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/06db1ab1/lang/cs/Org.Apache.REEF.IMRU/InProcess/InProcessIMRUClient.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.IMRU/InProcess/InProcessIMRUClient.cs b/lang/cs/Org.Apache.REEF.IMRU/InProcess/InProcessIMRUClient.cs index 77f7d9b..08cf201 100644 --- a/lang/cs/Org.Apache.REEF.IMRU/InProcess/InProcessIMRUClient.cs +++ b/lang/cs/Org.Apache.REEF.IMRU/InProcess/InProcessIMRUClient.cs @@ -40,14 +40,11 @@ namespace Org.Apache.REEF.IMRU.InProcess /// </summary> /// <remarks> /// This client assumes that all given Configurations can be merged in a conflict-free way. - /// </remarks> - /// <typeparam name="TMapInput">The type of the side information provided to the Map function</typeparam> - /// <typeparam name="TMapOutput">The return type of the Map function</typeparam> - /// <typeparam name="TResult">The return type of the computation.</typeparam> - public class InProcessIMRUClient<TMapInput, TMapOutput, TResult> : IIMRUClient<TMapInput, TMapOutput, TResult> + /// </remarks> + public class InProcessIMRUClient : IIMRUClient { private static readonly Logger Logger = - Logger.GetLogger(typeof (InProcessIMRUClient<TMapInput, TMapOutput, TResult>)); + Logger.GetLogger(typeof (InProcessIMRUClient)); private readonly int _numberOfMappers; @@ -65,9 +62,12 @@ namespace Org.Apache.REEF.IMRU.InProcess /// <summary> /// Submits the map job /// </summary> + /// <typeparam name="TMapInput">The type of the side information provided to the Map function</typeparam> + /// <typeparam name="TMapOutput">The return type of the Map function</typeparam> + /// <typeparam name="TResult">The return type of the computation.</typeparam> /// <param name="jobDefinition">Job definition given by the user</param> /// <returns>The result of the job</returns> - public IEnumerable<TResult> Submit(IMRUJobDefinition jobDefinition) + public IEnumerable<TResult> Submit<TMapInput, TMapOutput, TResult>(IMRUJobDefinition jobDefinition) { IConfiguration overallPerMapConfig = null; try @@ -91,7 +91,8 @@ namespace Org.Apache.REEF.IMRU.InProcess (ISet<IPerMapperConfigGenerator>) injector.GetNamedInstance(typeof (PerMapConfigGeneratorSet)); injector.BindVolatileInstance(GenericType<MapFunctions<TMapInput, TMapOutput>>.Class, - MakeMapFunctions(jobDefinition.MapFunctionConfiguration, jobDefinition.PartitionedDatasetConfiguration, perMapConfigGenerators)); + MakeMapFunctions<TMapInput, TMapOutput>(jobDefinition.MapFunctionConfiguration, + jobDefinition.PartitionedDatasetConfiguration, perMapConfigGenerators)); var runner = injector.GetInstance<IMRURunner<TMapInput, TMapOutput, TResult>>(); return runner.Run(); @@ -104,7 +105,7 @@ namespace Org.Apache.REEF.IMRU.InProcess /// <param name="partitionedDataSetConfig">Partitioned dataset configuration</param> /// <param name="perMapConfigGenerators">Per map configuration generators</param> /// <returns></returns> - private MapFunctions<TMapInput, TMapOutput> MakeMapFunctions(IConfiguration mapConfiguration, IConfiguration partitionedDataSetConfig, ISet<IPerMapperConfigGenerator> perMapConfigGenerators) + private MapFunctions<TMapInput, TMapOutput> MakeMapFunctions<TMapInput, TMapOutput>(IConfiguration mapConfiguration, IConfiguration partitionedDataSetConfig, ISet<IPerMapperConfigGenerator> perMapConfigGenerators) { IPartitionedDataSet dataset = TangFactory.GetTang().NewInjector(partitionedDataSetConfig).GetInstance<IPartitionedDataSet>(); http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/06db1ab1/lang/cs/Org.Apache.REEF.IMRU/InProcess/InProcessIMRUConfiguration.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.IMRU/InProcess/InProcessIMRUConfiguration.cs b/lang/cs/Org.Apache.REEF.IMRU/InProcess/InProcessIMRUConfiguration.cs index 6a2f01b..76bbe36 100644 --- a/lang/cs/Org.Apache.REEF.IMRU/InProcess/InProcessIMRUConfiguration.cs +++ b/lang/cs/Org.Apache.REEF.IMRU/InProcess/InProcessIMRUConfiguration.cs @@ -27,10 +27,7 @@ namespace Org.Apache.REEF.IMRU.InProcess /// <summary> /// Configuration module for the in-process IMRU. /// </summary> - /// <typeparam name="TMapInput">The type of the side information provided to the Map function</typeparam> - /// <typeparam name="TMapOutput">The return type of the Map function</typeparam> - /// <typeparam name="TResult">The return type of the computation.</typeparam> - public sealed class InProcessIMRUConfiguration<TMapInput, TMapOutput, TResult> : ConfigurationModuleBuilder + public sealed class InProcessIMRUConfiguration : ConfigurationModuleBuilder { /// <summary> /// The number of Mappers to instantiate. @@ -41,9 +38,9 @@ namespace Org.Apache.REEF.IMRU.InProcess /// Configuration module /// </summary> public static ConfigurationModule ConfigurationModule = - new InProcessIMRUConfiguration<TMapInput, TMapOutput, TResult>() - .BindImplementation(GenericType<IIMRUClient<TMapInput, TMapOutput, TResult>>.Class, - GenericType<InProcessIMRUClient<TMapInput, TMapOutput, TResult>>.Class) + new InProcessIMRUConfiguration() + .BindImplementation(GenericType<IIMRUClient>.Class, + GenericType<InProcessIMRUClient>.Class) .BindNamedParameter(GenericType<NumberOfMappers>.Class, NumberOfMappers) .Build(); } http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/06db1ab1/lang/cs/Org.Apache.REEF.IMRU/OnREEF/Client/REEFIMRUClient.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.IMRU/OnREEF/Client/REEFIMRUClient.cs b/lang/cs/Org.Apache.REEF.IMRU/OnREEF/Client/REEFIMRUClient.cs index 9e74f43..d178c4f 100644 --- a/lang/cs/Org.Apache.REEF.IMRU/OnREEF/Client/REEFIMRUClient.cs +++ b/lang/cs/Org.Apache.REEF.IMRU/OnREEF/Client/REEFIMRUClient.cs @@ -43,12 +43,9 @@ namespace Org.Apache.REEF.IMRU.OnREEF.Client /// <summary> /// Implements the IMRU client API on REEF. /// </summary> - /// <typeparam name="TMapInput">The type of the side information provided to the Map function</typeparam> - /// <typeparam name="TMapOutput">The return type of the Map function</typeparam> - /// <typeparam name="TResult">The return type of the computation.</typeparam> - internal sealed class REEFIMRUClient<TMapInput, TMapOutput, TResult> : IIMRUClient<TMapInput, TMapOutput, TResult> + internal sealed class REEFIMRUClient : IIMRUClient { - private static readonly Logger Logger = Logger.GetLogger(typeof(IMRUDriver<TMapInput, TMapOutput, TResult>)); + private static readonly Logger Logger = Logger.GetLogger(typeof (REEFIMRUClient)); private readonly IREEFClient _reefClient; private readonly JobSubmissionBuilderFactory _jobSubmissionBuilderFactory; @@ -66,13 +63,16 @@ namespace Org.Apache.REEF.IMRU.OnREEF.Client /// <summary> /// Submits the job to reefClient /// </summary> + /// <typeparam name="TMapInput">The type of the side information provided to the Map function</typeparam> + /// <typeparam name="TMapOutput">The return type of the Map function</typeparam> + /// <typeparam name="TResult">The return type of the computation.</typeparam> /// <param name="jobDefinition">IMRU job definition given by the user</param> /// <returns>Null as results will be later written to some directory</returns> - IEnumerable<TResult> IIMRUClient<TMapInput, TMapOutput, TResult>.Submit(IMRUJobDefinition jobDefinition) + IEnumerable<TResult> IIMRUClient.Submit<TMapInput, TMapOutput, TResult>(IMRUJobDefinition jobDefinition) { string driverId = string.Format("IMRU-{0}-Driver", jobDefinition.JobName); IConfiguration overallPerMapConfig = null; - + try { overallPerMapConfig = Configurations.Merge(jobDefinition.PerMapConfigGeneratorConfig.ToArray()); http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/06db1ab1/lang/cs/Org.Apache.REEF.IMRU/OnREEF/Client/REEFIMRUClientConfiguration.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.IMRU/OnREEF/Client/REEFIMRUClientConfiguration.cs b/lang/cs/Org.Apache.REEF.IMRU/OnREEF/Client/REEFIMRUClientConfiguration.cs index ac3bef3..0f3aa0a 100644 --- a/lang/cs/Org.Apache.REEF.IMRU/OnREEF/Client/REEFIMRUClientConfiguration.cs +++ b/lang/cs/Org.Apache.REEF.IMRU/OnREEF/Client/REEFIMRUClientConfiguration.cs @@ -27,15 +27,15 @@ namespace Org.Apache.REEF.IMRU.OnREEF.Client /// <summary> /// A configuration module for specifying REEFIMRUClient /// </summary> - public sealed class REEFIMRUClientConfiguration<TMapInput, TMapOutput, TResult> : ConfigurationModuleBuilder + public sealed class REEFIMRUClientConfiguration : ConfigurationModuleBuilder { /// <summary> /// Configuration module /// </summary> public static ConfigurationModule ConfigurationModule = - new REEFIMRUClientConfiguration<TMapInput, TMapOutput, TResult>() - .BindImplementation(GenericType<IIMRUClient<TMapInput, TMapOutput, TResult>>.Class, - GenericType<REEFIMRUClient<TMapInput, TMapOutput, TResult>>.Class) + new REEFIMRUClientConfiguration() + .BindImplementation(GenericType<IIMRUClient>.Class, + GenericType<REEFIMRUClient>.Class) .Build(); } } \ No newline at end of file
