Repository: reef Updated Branches: refs/heads/master 76cea200c -> c50f45bcf
[REEF-1201] Revert EvaluatorConfigurationsTests deleted from REEF-1189 This reverts "[REEF-1189] Delete EvaluatorConfigurationsTests and EvaluatorTests" This reverts commit a565695208c40b266083fc9ae797dd2cf731f1d9. JIRA: [REEF-1201](https://issues.apache.org/jira/browse/REEF-1201) Pull request: This closes #837 Project: http://git-wip-us.apache.org/repos/asf/reef/repo Commit: http://git-wip-us.apache.org/repos/asf/reef/commit/c50f45bc Tree: http://git-wip-us.apache.org/repos/asf/reef/tree/c50f45bc Diff: http://git-wip-us.apache.org/repos/asf/reef/diff/c50f45bc Branch: refs/heads/master Commit: c50f45bcfc539df7a32ad37fa5687d6063e56b8d Parents: 76cea20 Author: Mariia Mykhailova <[email protected]> Authored: Wed Feb 10 13:16:17 2016 -0800 Committer: Julia Wang <[email protected]> Committed: Wed Feb 10 16:29:52 2016 -0800 ---------------------------------------------------------------------- .../EvaluatorConfigurationsTests.cs | 185 +++++++++++++++++++ .../EvaluatorTests.cs | 96 ++++++++++ .../Org.Apache.REEF.Evaluator.Tests.csproj | 8 + .../ConfigFiles/evaluator.conf | Bin 0 -> 1862 bytes .../ConfigFiles/evaluatorWithService.conf | Bin 0 -> 6975 bytes .../Org.Apache.REEF.Examples.csproj | 10 +- .../Format/TestConfigurationModule.cs | 5 + .../Org.Apache.REEF.Tang.Tests.csproj | 5 +- 8 files changed, 307 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/reef/blob/c50f45bc/lang/cs/Org.Apache.REEF.Evaluator.Tests/EvaluatorConfigurationsTests.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Evaluator.Tests/EvaluatorConfigurationsTests.cs b/lang/cs/Org.Apache.REEF.Evaluator.Tests/EvaluatorConfigurationsTests.cs new file mode 100644 index 0000000..410a1dc --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Evaluator.Tests/EvaluatorConfigurationsTests.cs @@ -0,0 +1,185 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using Org.Apache.REEF.Common.Runtime.Evaluator.Utils; +using Org.Apache.REEF.Common.Services; +using Org.Apache.REEF.Common.Tasks; +using Org.Apache.REEF.Driver.Context; +using Org.Apache.REEF.Examples.HelloREEF; +using Org.Apache.REEF.Tang.Formats; +using Org.Apache.REEF.Tang.Implementations.Tang; +using Org.Apache.REEF.Tang.Interface; +using Org.Apache.REEF.Tang.Util; +using Org.Apache.REEF.Utilities.Logging; +using Org.Apache.REEF.Wake.Remote.Parameters; +using Org.Apache.REEF.Wake.StreamingCodec; +using Xunit; + +namespace Org.Apache.REEF.Evaluator.Tests +{ + public class EvaluatorConfigurationsTests + { + private static readonly Logger Logger = Logger.GetLogger(typeof(EvaluatorConfigurationsTests)); + private const string EvaluatorIdPrefix = "Node-"; + private const string ContextIdPrefix = "RootContext_"; + private const string RemoteIdPrefix = "socket://"; + private const string AppIdForTest = "REEF_LOCAL_RUNTIME"; + + [Fact] + [Trait("Priority", "0")] + [Trait("Category", "Unit")] + public void TestEvaluatorConfigurationFile() + { + AvroConfigurationSerializer serializer = new AvroConfigurationSerializer(); + var avroConfiguration = serializer.AvroDeserializeFromFile("evaluator.conf"); + + Assert.NotNull(avroConfiguration); + Assert.Equal(avroConfiguration.language, Language.Java.ToString()); + + foreach (var b in avroConfiguration.Bindings) + { + Logger.Log(Level.Info, "Key = " + b.key + " Value = " + b.value); + } + } + + [Fact] + [Trait("Priority", "0")] + [Trait("Category", "Unit")] + public void TestDeserializationWithAlias() + { + AvroConfigurationSerializer serializer = new AvroConfigurationSerializer(); + var avroConfiguration = serializer.AvroDeserializeFromFile("evaluator.conf"); + var language = avroConfiguration.language; + Assert.True(language.ToString().Equals(Language.Java.ToString())); + + var classHierarchy = TangFactory.GetTang() + .GetClassHierarchy(new string[] { typeof(ApplicationIdentifier).Assembly.GetName().Name }); + var config = serializer.FromAvro(avroConfiguration, classHierarchy); + + IInjector evaluatorInjector = TangFactory.GetTang().NewInjector(config); + string appid = evaluatorInjector.GetNamedInstance<ApplicationIdentifier, string>(); + string remoteId = evaluatorInjector.GetNamedInstance<DriverRemoteIdentifier, string>(); + + string evaluatorIdentifier = evaluatorInjector.GetNamedInstance<EvaluatorIdentifier, string>(); + string rid = evaluatorInjector.GetNamedInstance<ErrorHandlerRid, string>(); + string launchId = evaluatorInjector.GetNamedInstance<LaunchId, string>(); + + Assert.True(remoteId.StartsWith(RemoteIdPrefix)); + Assert.True(appid.Equals(AppIdForTest)); + Assert.True(evaluatorIdentifier.StartsWith(EvaluatorIdPrefix)); + Assert.True(rid.StartsWith(RemoteIdPrefix)); + Assert.True(launchId.Equals(AppIdForTest)); + } + + /// <summary> + /// This test is to deserialize a evaluator configuration file using alias if the parameter cannot be + /// found in the class hierarchy. The config file used in the test was generated when running HelloRREEF. + /// It contains task and context configuration strings. + /// </summary> + [Fact] + [Trait("Priority", "0")] + [Trait("Category", "Unit")] + public void TestDeserializationForContextAndTask() + { + AvroConfigurationSerializer serializer = new AvroConfigurationSerializer(); + + var classHierarchy = TangFactory.GetTang() + .GetClassHierarchy(new string[] { typeof(ApplicationIdentifier).Assembly.GetName().Name }); + var config = serializer.FromFile("evaluator.conf", classHierarchy); + + IInjector evaluatorInjector = TangFactory.GetTang().NewInjector(config); + + string taskConfigString = evaluatorInjector.GetNamedInstance<InitialTaskConfiguration, string>(); + string contextConfigString = evaluatorInjector.GetNamedInstance<RootContextConfiguration, string>(); + + var contextClassHierarchy = TangFactory.GetTang().GetClassHierarchy(new string[] + { + typeof(ContextConfigurationOptions.ContextIdentifier).Assembly.GetName().Name + }); + var contextConfig = serializer.FromString(contextConfigString, contextClassHierarchy); + + var taskClassHierarchy = TangFactory.GetTang().GetClassHierarchy(new string[] + { + typeof(ITask).Assembly.GetName().Name, + typeof(HelloTask).Assembly.GetName().Name + }); + var taskConfig = serializer.FromString(taskConfigString, taskClassHierarchy); + + var contextInjector = evaluatorInjector.ForkInjector(contextConfig); + string contextId = contextInjector.GetNamedInstance<ContextConfigurationOptions.ContextIdentifier, string>(); + Assert.True(contextId.StartsWith(ContextIdPrefix)); + + var taskInjector = contextInjector.ForkInjector(taskConfig); + + string taskId = taskInjector.GetNamedInstance<TaskConfigurationOptions.Identifier, string>(); + ITask task = taskInjector.GetInstance<ITask>(); + Assert.True(taskId.StartsWith("HelloTask")); + Assert.True(task is HelloTask); + } + + /// <summary> + /// This test is to deserialize a evaluator configuration file using alias if the parameter cannot be + /// found in the class hierarchy. The config file used in the test was generated when running TestBroadCastReduceOperators. + /// It contains service and context configuration strings. + /// </summary> + [Fact] + [Trait("Priority", "0")] + [Trait("Category", "Unit")] + public void TestDeserializationForServiceAndContext() + { + AvroConfigurationSerializer serializer = new AvroConfigurationSerializer(); + + var classHierarchy = TangFactory.GetTang() + .GetClassHierarchy(new string[] { typeof(ApplicationIdentifier).Assembly.GetName().Name }); + var config = serializer.FromFile("evaluatorWithService.conf", classHierarchy); + + IInjector evaluatorInjector = TangFactory.GetTang().NewInjector(config); + + string contextConfigString = evaluatorInjector.GetNamedInstance<RootContextConfiguration, string>(); + string rootServiceConfigString = evaluatorInjector.GetNamedInstance<RootServiceConfiguration, string>(); + + var contextClassHierarchy = TangFactory.GetTang().GetClassHierarchy(new string[] + { + typeof(ContextConfigurationOptions.ContextIdentifier).Assembly.GetName().Name + }); + + var contextConfig = serializer.FromString(contextConfigString, contextClassHierarchy); + + var serviceClassHierarchy = TangFactory.GetTang().GetClassHierarchy(new string[] + { + typeof(ServicesConfigurationOptions).Assembly.GetName().Name, + typeof(IStreamingCodec<>).Assembly.GetName().Name + }); + var rootServiceConfig = serializer.FromString(rootServiceConfigString, serviceClassHierarchy); + + var contextInjector = evaluatorInjector.ForkInjector(contextConfig); + string contextId = contextInjector.GetNamedInstance<ContextConfigurationOptions.ContextIdentifier, string>(); + Assert.True(contextId.StartsWith("MasterTaskContext")); + + string serviceConfigString = TangFactory.GetTang().NewInjector(rootServiceConfig) + .GetNamedInstance<ServicesConfigurationOptions.ServiceConfigString, string>(); + + var serviceConfig = serializer.FromString(serviceConfigString, serviceClassHierarchy); + + var serviceInjector = contextInjector.ForkInjector(serviceConfig); + var tcpCountRange = serviceInjector.GetNamedInstance<TcpPortRangeStart, int>(); + var tcpCountCount = serviceInjector.GetNamedInstance<TcpPortRangeCount, int>(); + Assert.True(tcpCountRange > 0); + Assert.True(tcpCountCount > 0); + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/reef/blob/c50f45bc/lang/cs/Org.Apache.REEF.Evaluator.Tests/EvaluatorTests.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Evaluator.Tests/EvaluatorTests.cs b/lang/cs/Org.Apache.REEF.Evaluator.Tests/EvaluatorTests.cs new file mode 100644 index 0000000..3781699 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Evaluator.Tests/EvaluatorTests.cs @@ -0,0 +1,96 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System; +using System.IO; +using Org.Apache.REEF.Common.Avro; +using Org.Apache.REEF.Common.Evaluator; +using Org.Apache.REEF.Common.Tasks; +using Org.Apache.REEF.Examples.Tasks.ShellTask; +using Org.Apache.REEF.Tang.Formats; +using Org.Apache.REEF.Tang.Formats.AvroConfigurationDataContract; +using Org.Apache.REEF.Tang.Implementations.Tang; +using Org.Apache.REEF.Tang.Interface; +using Org.Apache.REEF.Tang.Util; +using Xunit; + +namespace Org.Apache.REEF.Evaluator.Tests +{ + public class EvaluatorTests + { + [Fact] + [Trait("Priority", "0")] + [Trait("Category", "Functional")] + [Trait("Description", "Parse Evaluator configuration from Java, inject and execute Shell task with DIR command based on the configuration")] + public void CanInjectAndExecuteTask() + { + // to enforce that shell task dll be copied to output directory. + ShellTask tmpTask = new ShellTask("invalid"); + Assert.NotNull(tmpTask); + + string tmp = Directory.GetCurrentDirectory(); + Assert.NotNull(tmp); + + AvroConfigurationSerializer serializer = new AvroConfigurationSerializer(); + AvroConfiguration avroConfiguration = serializer.AvroDeserializeFromFile("evaluator.conf"); + Assert.NotNull(avroConfiguration); + + ICsConfigurationBuilder cb = TangFactory.GetTang().NewConfigurationBuilder(); + cb.AddConfiguration(TaskConfiguration.ConfigurationModule + .Set(TaskConfiguration.Identifier, "Test_CLRContext_task") + .Set(TaskConfiguration.Task, GenericType<ShellTask>.Class) + .Build()); + cb.BindNamedParameter<ShellTask.Command, string>(GenericType<ShellTask.Command>.Class, "dir"); + + IConfiguration taskConfiguration = cb.Build(); + + string taskConfig = serializer.ToString(taskConfiguration); + + ITask task = null; + TaskConfiguration config = new TaskConfiguration(taskConfig); + Assert.NotNull(config); + try + { + IInjector injector = TangFactory.GetTang().NewInjector(config.TangConfig); + task = (ITask)injector.GetInstance(typeof(ITask)); + } + catch (Exception e) + { + throw new InvalidOperationException("unable to inject task with configuration: " + taskConfig, e); + } + + byte[] bytes = task.Call(null); + string result = System.Text.Encoding.Default.GetString(bytes); + + // a dir command is executed in the container directory, which includes the file "evaluator.conf" + Assert.True(result.Contains("evaluator.conf")); + } + + [Fact] + [Trait("Priority", "0")] + [Trait("Category", "Unit")] + [Trait("Description", "Test driver information extracted from Http server")] + public void CanExtractDriverInformation() + { + const string infoString = "{\"remoteId\":\"socket://10.121.136.231:14272\",\"startTime\":\"2014 08 28 10:50:32\",\"services\":[{\"serviceName\":\"NameServer\",\"serviceInfo\":\"10.121.136.231:16663\"}]}"; + AvroDriverInfo info = AvroJsonSerializer<AvroDriverInfo>.FromString(infoString); + Assert.True(info.remoteId.Equals("socket://10.121.136.231:14272")); + Assert.True(info.startTime.Equals("2014 08 28 10:50:32")); + Assert.True(new DriverInformation(info.remoteId, info.startTime, info.services).NameServerId.Equals("10.121.136.231:16663")); + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/reef/blob/c50f45bc/lang/cs/Org.Apache.REEF.Evaluator.Tests/Org.Apache.REEF.Evaluator.Tests.csproj ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Evaluator.Tests/Org.Apache.REEF.Evaluator.Tests.csproj b/lang/cs/Org.Apache.REEF.Evaluator.Tests/Org.Apache.REEF.Evaluator.Tests.csproj index 3db8ec2..da0fe33 100644 --- a/lang/cs/Org.Apache.REEF.Evaluator.Tests/Org.Apache.REEF.Evaluator.Tests.csproj +++ b/lang/cs/Org.Apache.REEF.Evaluator.Tests/Org.Apache.REEF.Evaluator.Tests.csproj @@ -60,6 +60,8 @@ under the License. </ItemGroup> <ItemGroup> <Compile Include="ContextRuntimeTests.cs" /> + <Compile Include="EvaluatorConfigurationsTests.cs" /> + <Compile Include="EvaluatorTests.cs" /> <Compile Include="Properties\AssemblyInfo.cs" /> </ItemGroup> <ItemGroup> @@ -97,6 +99,12 @@ under the License. </ProjectReference> </ItemGroup> <ItemGroup> + <None Include="$(SolutionDir)\Org.Apache.REEF.Examples\ConfigFiles\evaluator.conf"> + <CopyToOutputDirectory>Always</CopyToOutputDirectory> + </None> + <None Include="$(SolutionDir)\Org.Apache.REEF.Examples\ConfigFiles\evaluatorWithService.conf"> + <CopyToOutputDirectory>Always</CopyToOutputDirectory> + </None> <None Include="packages.config" /> </ItemGroup> <Import Project="$(VSToolsPath)\TeamTest\Microsoft.TestTools.targets" Condition="Exists('$(VSToolsPath)\TeamTest\Microsoft.TestTools.targets')" /> http://git-wip-us.apache.org/repos/asf/reef/blob/c50f45bc/lang/cs/Org.Apache.REEF.Examples/ConfigFiles/evaluator.conf ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Examples/ConfigFiles/evaluator.conf b/lang/cs/Org.Apache.REEF.Examples/ConfigFiles/evaluator.conf new file mode 100644 index 0000000..20c2016 Binary files /dev/null and b/lang/cs/Org.Apache.REEF.Examples/ConfigFiles/evaluator.conf differ http://git-wip-us.apache.org/repos/asf/reef/blob/c50f45bc/lang/cs/Org.Apache.REEF.Examples/ConfigFiles/evaluatorWithService.conf ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Examples/ConfigFiles/evaluatorWithService.conf b/lang/cs/Org.Apache.REEF.Examples/ConfigFiles/evaluatorWithService.conf new file mode 100644 index 0000000..8b91c06 Binary files /dev/null and b/lang/cs/Org.Apache.REEF.Examples/ConfigFiles/evaluatorWithService.conf differ http://git-wip-us.apache.org/repos/asf/reef/blob/c50f45bc/lang/cs/Org.Apache.REEF.Examples/Org.Apache.REEF.Examples.csproj ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Examples/Org.Apache.REEF.Examples.csproj b/lang/cs/Org.Apache.REEF.Examples/Org.Apache.REEF.Examples.csproj index cd409a3..17d36e3 100644 --- a/lang/cs/Org.Apache.REEF.Examples/Org.Apache.REEF.Examples.csproj +++ b/lang/cs/Org.Apache.REEF.Examples/Org.Apache.REEF.Examples.csproj @@ -90,6 +90,14 @@ under the License. <Name>Org.Apache.REEF.Wake</Name> </ProjectReference> </ItemGroup> + <ItemGroup> + <None Include="ConfigFiles\evaluator.conf"> + <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> + </None> + <None Include="ConfigFiles\evaluatorWithService.conf"> + <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> + </None> + </ItemGroup> <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" /> <Import Project="$(SolutionDir)\.nuget\NuGet.targets" Condition="Exists('$(SolutionDir)\.nuget\NuGet.targets')" /> <Import Project="$(PackagesDir)\StyleCop.MSBuild.4.7.49.1\build\StyleCop.MSBuild.Targets" Condition="Exists('$(PackagesDir)\StyleCop.MSBuild.4.7.49.1\build\StyleCop.MSBuild.Targets')" /> @@ -100,4 +108,4 @@ under the License. <Target Name="AfterBuild"> </Target> --> -</Project> \ No newline at end of file +</Project> http://git-wip-us.apache.org/repos/asf/reef/blob/c50f45bc/lang/cs/Org.Apache.REEF.Tang.Tests/Format/TestConfigurationModule.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Tang.Tests/Format/TestConfigurationModule.cs b/lang/cs/Org.Apache.REEF.Tang.Tests/Format/TestConfigurationModule.cs index 2e96240..67f3076 100644 --- a/lang/cs/Org.Apache.REEF.Tang.Tests/Format/TestConfigurationModule.cs +++ b/lang/cs/Org.Apache.REEF.Tang.Tests/Format/TestConfigurationModule.cs @@ -85,6 +85,11 @@ namespace Org.Apache.REEF.Tang.Tests.Format var c5 = serializerImpl.FromFile("TangTestCs1.avro"); IFoo f5 = (IFoo)TangFactory.GetTang().NewInjector(c5).GetInstance(fooType); Assert.Equal(f5.getFooness(), 12); + + // this is to test the file generated from Java. name,value b=must be recognized by C# class hierarchy + AvroConfigurationSerializer serializer = new AvroConfigurationSerializer(); + var avroConfig = serializer.AvroDeserializeFromFile("Evaluator.conf"); + Assert.NotNull(avroConfig); } [Fact] http://git-wip-us.apache.org/repos/asf/reef/blob/c50f45bc/lang/cs/Org.Apache.REEF.Tang.Tests/Org.Apache.REEF.Tang.Tests.csproj ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Tang.Tests/Org.Apache.REEF.Tang.Tests.csproj b/lang/cs/Org.Apache.REEF.Tang.Tests/Org.Apache.REEF.Tang.Tests.csproj index dbcf552..7cfd196 100644 --- a/lang/cs/Org.Apache.REEF.Tang.Tests/Org.Apache.REEF.Tang.Tests.csproj +++ b/lang/cs/Org.Apache.REEF.Tang.Tests/Org.Apache.REEF.Tang.Tests.csproj @@ -130,6 +130,9 @@ under the License. <Compile Include="Utilities\Utilities.cs" /> </ItemGroup> <ItemGroup> + <None Include="$(SolutionDir)\Org.Apache.REEF.Examples\ConfigFiles\evaluator.conf"> + <CopyToOutputDirectory>Always</CopyToOutputDirectory> + </None> <None Include="packages.config"> <SubType>Designer</SubType> </None> @@ -179,4 +182,4 @@ under the License. <Target Name="AfterBuild"> </Target> --> -</Project> \ No newline at end of file +</Project>
