Repository: reef Updated Branches: refs/heads/master 07874cbb6 -> 00ac1d06a
[REEF-1232] Fix binding in TaskConfiguration for ITaskMessageSource * replace the binding in TaskConfiguration for ITaskMessageSource from an interface to a set of interface * Update TestTaskMessage classes to verify the messages back forth between driver and task * Update test bases class for evaluator log validation as well. JIRA: [REEF-1232](https://issues.apache.org/jira/browse/REEF-1232) Pull Request This closes #871 Project: http://git-wip-us.apache.org/repos/asf/reef/repo Commit: http://git-wip-us.apache.org/repos/asf/reef/commit/00ac1d06 Tree: http://git-wip-us.apache.org/repos/asf/reef/tree/00ac1d06 Diff: http://git-wip-us.apache.org/repos/asf/reef/diff/00ac1d06 Branch: refs/heads/master Commit: 00ac1d06a144f92f519ae53cf960e88827034a4a Parents: 07874cb Author: Julia Wang <[email protected]> Authored: Wed Mar 2 16:01:54 2016 -0800 Committer: Markus Weimer <[email protected]> Committed: Thu Mar 3 11:06:37 2016 -0800 ---------------------------------------------------------------------- .../Tasks/TaskConfiguration.cs | 2 +- .../Functional/Bridge/TestContextStack.cs | 4 +- .../Bridge/TestFailedEvaluatorEventHandler.cs | 2 +- .../Bridge/TestFailedTaskEventHandler.cs | 2 +- .../Functional/Bridge/TestSimpleContext.cs | 2 +- .../Bridge/TestSimpleEventHandlers.cs | 2 +- .../Functional/Messaging/MessageTask.cs | 6 +-- .../Functional/Messaging/TestTaskMessage.cs | 17 ++++++- .../Functional/ReefFunctionalTest.cs | 51 ++++++++++++++------ .../Functional/RuntimeName/RuntimeNameTest.cs | 2 +- 10 files changed, 62 insertions(+), 28 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/reef/blob/00ac1d06/lang/cs/Org.Apache.REEF.Common/Tasks/TaskConfiguration.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Common/Tasks/TaskConfiguration.cs b/lang/cs/Org.Apache.REEF.Common/Tasks/TaskConfiguration.cs index 409a0de..a0f606e 100644 --- a/lang/cs/Org.Apache.REEF.Common/Tasks/TaskConfiguration.cs +++ b/lang/cs/Org.Apache.REEF.Common/Tasks/TaskConfiguration.cs @@ -94,7 +94,7 @@ namespace Org.Apache.REEF.Common.Tasks { return new TaskConfiguration() .BindImplementation(GenericType<ITask>.Class, Task) - .BindImplementation(GenericType<ITaskMessageSource>.Class, OnSendMessage) + .BindSetEntry(GenericType<TaskConfigurationOptions.TaskMessageSources>.Class, OnSendMessage) .BindImplementation(GenericType<IDriverMessageHandler>.Class, OnMessage) .BindImplementation(GenericType<IDriverConnectionMessageHandler>.Class, OnDriverConnectionChanged) .BindNamedParameter(GenericType<TaskConfigurationOptions.Identifier>.Class, Identifier) http://git-wip-us.apache.org/repos/asf/reef/blob/00ac1d06/lang/cs/Org.Apache.REEF.Tests/Functional/Bridge/TestContextStack.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Tests/Functional/Bridge/TestContextStack.cs b/lang/cs/Org.Apache.REEF.Tests/Functional/Bridge/TestContextStack.cs index 2c2daa6..f756a36 100644 --- a/lang/cs/Org.Apache.REEF.Tests/Functional/Bridge/TestContextStack.cs +++ b/lang/cs/Org.Apache.REEF.Tests/Functional/Bridge/TestContextStack.cs @@ -57,8 +57,8 @@ namespace Org.Apache.REEF.Tests.Functional.Bridge CleanUp(testFolder); TestRun(DriverConfigurations(), typeof(ContextStackHandlers), 1, "testContextStack", "local", testFolder); ValidateSuccessForLocalRuntime(2, testFolder: testFolder); - ValidateMessageSuccessfullyLogged(TaskValidationMessage, testFolder); - ValidateMessageSuccessfullyLogged(ClosedContextValidationMessage, testFolder); + ValidateMessageSuccessfullyLoggedForDriver(TaskValidationMessage, testFolder); + ValidateMessageSuccessfullyLoggedForDriver(ClosedContextValidationMessage, testFolder); CleanUp(testFolder); } http://git-wip-us.apache.org/repos/asf/reef/blob/00ac1d06/lang/cs/Org.Apache.REEF.Tests/Functional/Bridge/TestFailedEvaluatorEventHandler.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Tests/Functional/Bridge/TestFailedEvaluatorEventHandler.cs b/lang/cs/Org.Apache.REEF.Tests/Functional/Bridge/TestFailedEvaluatorEventHandler.cs index 755c92c..c7a01c0 100644 --- a/lang/cs/Org.Apache.REEF.Tests/Functional/Bridge/TestFailedEvaluatorEventHandler.cs +++ b/lang/cs/Org.Apache.REEF.Tests/Functional/Bridge/TestFailedEvaluatorEventHandler.cs @@ -50,7 +50,7 @@ namespace Org.Apache.REEF.Tests.Functional.Bridge CleanUp(testFolder); TestRun(DriverConfigurations(), typeof(FailedEvaluatorDriver), 1, "failedEvaluatorTest", "local", testFolder); ValidateSuccessForLocalRuntime(0, numberOfEvaluatorsToFail: 1, testFolder: testFolder); - ValidateMessageSuccessfullyLogged(FailedEvaluatorMessage, testFolder); + ValidateMessageSuccessfullyLoggedForDriver(FailedEvaluatorMessage, testFolder); CleanUp(testFolder); } http://git-wip-us.apache.org/repos/asf/reef/blob/00ac1d06/lang/cs/Org.Apache.REEF.Tests/Functional/Bridge/TestFailedTaskEventHandler.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Tests/Functional/Bridge/TestFailedTaskEventHandler.cs b/lang/cs/Org.Apache.REEF.Tests/Functional/Bridge/TestFailedTaskEventHandler.cs index 82e796a..db1001b 100644 --- a/lang/cs/Org.Apache.REEF.Tests/Functional/Bridge/TestFailedTaskEventHandler.cs +++ b/lang/cs/Org.Apache.REEF.Tests/Functional/Bridge/TestFailedTaskEventHandler.cs @@ -51,7 +51,7 @@ namespace Org.Apache.REEF.Tests.Functional.Bridge CleanUp(testFolder); TestRun(DriverConfigurations(), typeof(FailedTaskDriver), 1, "failedTaskTest", "local", testFolder); ValidateSuccessForLocalRuntime(numberOfContextsToClose: 1, numberOfTasksToFail: 1, testFolder: testFolder); - ValidateMessageSuccessfullyLogged(FailedTaskMessage, testFolder); + ValidateMessageSuccessfullyLoggedForDriver(FailedTaskMessage, testFolder); CleanUp(testFolder); } http://git-wip-us.apache.org/repos/asf/reef/blob/00ac1d06/lang/cs/Org.Apache.REEF.Tests/Functional/Bridge/TestSimpleContext.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Tests/Functional/Bridge/TestSimpleContext.cs b/lang/cs/Org.Apache.REEF.Tests/Functional/Bridge/TestSimpleContext.cs index 44c5fe0..d0da1ba 100644 --- a/lang/cs/Org.Apache.REEF.Tests/Functional/Bridge/TestSimpleContext.cs +++ b/lang/cs/Org.Apache.REEF.Tests/Functional/Bridge/TestSimpleContext.cs @@ -64,7 +64,7 @@ namespace Org.Apache.REEF.Tests.Functional.Bridge CleanUp(testFolder); TestRun(configuration, typeof(TestContextHandlers), 1, "testSimpleContext", "local", testFolder); ValidateSuccessForLocalRuntime(1, testFolder: testFolder); - ValidateMessageSuccessfullyLogged(ValidationMessage, testFolder); + ValidateMessageSuccessfullyLoggedForDriver(ValidationMessage, testFolder); CleanUp(testFolder); } http://git-wip-us.apache.org/repos/asf/reef/blob/00ac1d06/lang/cs/Org.Apache.REEF.Tests/Functional/Bridge/TestSimpleEventHandlers.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Tests/Functional/Bridge/TestSimpleEventHandlers.cs b/lang/cs/Org.Apache.REEF.Tests/Functional/Bridge/TestSimpleEventHandlers.cs index b652065..f8b5dbe 100644 --- a/lang/cs/Org.Apache.REEF.Tests/Functional/Bridge/TestSimpleEventHandlers.cs +++ b/lang/cs/Org.Apache.REEF.Tests/Functional/Bridge/TestSimpleEventHandlers.cs @@ -50,7 +50,7 @@ namespace Org.Apache.REEF.Tests.Functional.Bridge CleanUp(testFolder); TestRun(DriverConfigurations(), typeof(HelloSimpleEventHandlers), 2, "simpleHandler", "local", testFolder); ValidateSuccessForLocalRuntime(1, testFolder: testFolder); - ValidateMessageSuccessfullyLogged("Evaluator is assigned with 3072 MB of memory and 1 cores.", testFolder); + ValidateMessageSuccessfullyLoggedForDriver("Evaluator is assigned with 3072 MB of memory and 1 cores.", testFolder); CleanUp(testFolder); } http://git-wip-us.apache.org/repos/asf/reef/blob/00ac1d06/lang/cs/Org.Apache.REEF.Tests/Functional/Messaging/MessageTask.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Tests/Functional/Messaging/MessageTask.cs b/lang/cs/Org.Apache.REEF.Tests/Functional/Messaging/MessageTask.cs index 5ed25a9..6f2785d 100644 --- a/lang/cs/Org.Apache.REEF.Tests/Functional/Messaging/MessageTask.cs +++ b/lang/cs/Org.Apache.REEF.Tests/Functional/Messaging/MessageTask.cs @@ -44,10 +44,11 @@ namespace Org.Apache.REEF.Tests.Functional.Messaging public Optional<TaskMessage> Message { get - { + { TaskMessage defaultTaskMessage = TaskMessage.From( "messagingSourceId", ByteUtilities.StringToByteArrays(MessageSend + " generated at " + DateTime.Now.ToString(CultureInfo.InvariantCulture))); + LOGGER.Log(Level.Info, "Message is sent back from task to driver:" + defaultTaskMessage.Message); return Optional<TaskMessage>.Of(defaultTaskMessage); } @@ -70,7 +71,6 @@ namespace Org.Apache.REEF.Tests.Functional.Messaging private void DriverMessage(string message) { - LOGGER.Log(Level.Info, "Received DriverMessage in TaskMsg: " + message); if (!message.Equals(MessageDriver.Message)) { Exceptions.Throw(new Exception("Unexpected driver message: " + message), "Unexpected driver message received: " + message, LOGGER); @@ -90,10 +90,10 @@ namespace Org.Apache.REEF.Tests.Functional.Messaging public void Handle(IDriverMessage value) { string message = string.Empty; - LOGGER.Log(Level.Verbose, "Received a message from driver, handling it with MessagingDriverMessageHandler"); if (value.Message.IsPresent()) { message = ByteUtilities.ByteArraysToString(value.Message.Value); + LOGGER.Log(Level.Info, "Received a message from driver, handling it with MessagingDriverMessageHandler:" + message); } _parentTask.DriverMessage(message); } http://git-wip-us.apache.org/repos/asf/reef/blob/00ac1d06/lang/cs/Org.Apache.REEF.Tests/Functional/Messaging/TestTaskMessage.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Tests/Functional/Messaging/TestTaskMessage.cs b/lang/cs/Org.Apache.REEF.Tests/Functional/Messaging/TestTaskMessage.cs index 65e91fe..9152eb8 100644 --- a/lang/cs/Org.Apache.REEF.Tests/Functional/Messaging/TestTaskMessage.cs +++ b/lang/cs/Org.Apache.REEF.Tests/Functional/Messaging/TestTaskMessage.cs @@ -15,6 +15,9 @@ // specific language governing permissions and limitations // under the License. +using System; +using System.Collections.Generic; +using System.Runtime.InteropServices; using Org.Apache.REEF.Driver; using Org.Apache.REEF.Driver.Bridge; using Org.Apache.REEF.Driver.Defaults; @@ -48,10 +51,22 @@ namespace Org.Apache.REEF.Tests.Functional.Messaging //// TODO[JIRA REEF-1184]: add timeout 180 sec public void TestSendTaskMessage() { - string testFolder = DefaultRuntimeFolder + TestNumber++; + string testFolder = DefaultRuntimeFolder + Guid.NewGuid(); CleanUp(testFolder); TestRun(DriverConfigurations(), typeof(MessageDriver), 1, "simpleHandler", "local", testFolder); ValidateSuccessForLocalRuntime(1, testFolder: testFolder); + + var messages = new List<string>(); + messages.Add("TaskMessagingTaskMessageHandler received following message from Task:"); + messages.Add("Message: MESSAGE:TASK generated"); + messages.Add("is to send message MESSAGE::DRIVER"); + ValidateMessageSuccessfullyLogged(messages, "driver", DriverStdout, testFolder, 0); + + var messages2 = new List<string>(); + messages.Add("Received a message from driver, handling it with MessagingDriverMessageHandler:MESSAGE::DRIVER"); + messages.Add("Message is sent back from task to driver:"); + ValidateMessageSuccessfullyLogged(messages2, "Node-*", EvaluatorStdout, testFolder, 0); + CleanUp(testFolder); } http://git-wip-us.apache.org/repos/asf/reef/blob/00ac1d06/lang/cs/Org.Apache.REEF.Tests/Functional/ReefFunctionalTest.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Tests/Functional/ReefFunctionalTest.cs b/lang/cs/Org.Apache.REEF.Tests/Functional/ReefFunctionalTest.cs index e547988..9250e67 100644 --- a/lang/cs/Org.Apache.REEF.Tests/Functional/ReefFunctionalTest.cs +++ b/lang/cs/Org.Apache.REEF.Tests/Functional/ReefFunctionalTest.cs @@ -16,6 +16,7 @@ // under the License. using System; +using System.Collections.Generic; using System.Globalization; using System.IO; using System.Linq; @@ -39,10 +40,11 @@ namespace Org.Apache.REEF.Tests.Functional { public class ReefFunctionalTest : IDisposable { - protected const string _stdout = "driver.stdout"; - protected const string _stderr = "driver.stderr"; - protected const string _cmdFile = "run.cmd"; - protected const string _binFolder = "."; + protected const string DriverStdout = "driver.stdout"; + protected const string DriverStderr = "driver.stderr"; + protected const string EvaluatorStdout = "evaluator.stdout"; + protected const string CmdFile = "run.cmd"; + protected const string BinFolder = "."; protected static int TestNumber = 1; protected const string DefaultRuntimeFolder = "REEF_LOCAL_RUNTIME"; @@ -99,9 +101,9 @@ namespace Org.Apache.REEF.Tests.Functional ValidationUtilities.ValidateEnvVariable("JAVA_HOME"); - if (!Directory.Exists(_binFolder)) + if (!Directory.Exists(BinFolder)) { - throw new InvalidOperationException(_binFolder + " not found in current directory, cannot init test"); + throw new InvalidOperationException(BinFolder + " not found in current directory, cannot init test"); } } @@ -148,7 +150,7 @@ namespace Org.Apache.REEF.Tests.Functional { try { - lines = File.ReadAllLines(GetLogFile(_stdout, testFolder)); + lines = File.ReadAllLines(GetLogFile(DriverStdout, "driver", testFolder)); break; } catch (Exception) @@ -174,14 +176,21 @@ namespace Org.Apache.REEF.Tests.Functional } } - protected void ValidateMessageSuccessfullyLogged(string message, string testFolder, int numberOfoccurances = 1) + protected void ValidateMessageSuccessfullyLoggedForDriver(string message, string testFolder, int numberOfoccurances = 1) + { + var msgs = new List<string>(); + msgs.Add(message); + ValidateMessageSuccessfullyLogged(msgs, "driver", DriverStdout, testFolder, numberOfoccurances); + } + + protected void ValidateMessageSuccessfullyLogged(IList<string> messages, string subfolder, string fileName, string testFolder, int numberOfoccurances = 1) { string[] lines = null; for (int i = 0; i < 60; i++) { try { - lines = File.ReadAllLines(GetLogFile(_stdout, testFolder)); + lines = File.ReadAllLines(GetLogFile(fileName, subfolder, testFolder)); break; } catch (Exception) @@ -192,8 +201,18 @@ namespace Org.Apache.REEF.Tests.Functional if (lines != null) { - string[] successIndicators = lines.Where(s => s.Contains(message)).ToArray(); - Assert.Equal(numberOfoccurances, successIndicators.Count()); + foreach (string message in messages) + { + string[] successIndicators = lines.Where(s => s.Contains(message)).ToArray(); + if (numberOfoccurances > 0) + { + Assert.Equal(numberOfoccurances, successIndicators.Count()); + } + else + { + Assert.NotEqual(0, successIndicators.Count()); + } + } } else { @@ -214,9 +233,9 @@ namespace Org.Apache.REEF.Tests.Functional } } - protected string GetLogFile(string logFileName, string testFolder = DefaultRuntimeFolder) + protected string GetLogFile(string logFileName, string subfolder = "driver", string testFolder = DefaultRuntimeFolder) { - string driverContainerDirectory = Directory.GetDirectories(Path.Combine(Directory.GetCurrentDirectory(), testFolder), "driver", SearchOption.AllDirectories).SingleOrDefault(); + string driverContainerDirectory = Directory.GetDirectories(Path.Combine(Directory.GetCurrentDirectory(), testFolder), subfolder, SearchOption.AllDirectories).SingleOrDefault(); Console.WriteLine("GetLogFile, driverContainerDirectory:" + driverContainerDirectory); if (string.IsNullOrWhiteSpace(driverContainerDirectory)) @@ -226,15 +245,15 @@ namespace Org.Apache.REEF.Tests.Functional string logFile = Path.Combine(driverContainerDirectory, logFileName); if (!File.Exists(logFile)) { - throw new InvalidOperationException("Driver stdout file not found: " + logFile); + throw new InvalidOperationException("Log file not found: " + logFile); } return logFile; } private void UploadDriverLog() { - string driverStdout = GetLogFile(_stdout); - string driverStderr = GetLogFile(_stderr); + string driverStdout = GetLogFile(DriverStdout); + string driverStderr = GetLogFile(DriverStderr); CloudStorageAccount storageAccount = CloudStorageAccount.Parse(GetStorageConnectionString()); CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); CloudBlobContainer container = blobClient.GetContainerReference(DateTime.Now.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)); http://git-wip-us.apache.org/repos/asf/reef/blob/00ac1d06/lang/cs/Org.Apache.REEF.Tests/Functional/RuntimeName/RuntimeNameTest.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Tests/Functional/RuntimeName/RuntimeNameTest.cs b/lang/cs/Org.Apache.REEF.Tests/Functional/RuntimeName/RuntimeNameTest.cs index 7e65596..fa08054 100644 --- a/lang/cs/Org.Apache.REEF.Tests/Functional/RuntimeName/RuntimeNameTest.cs +++ b/lang/cs/Org.Apache.REEF.Tests/Functional/RuntimeName/RuntimeNameTest.cs @@ -50,7 +50,7 @@ namespace Org.Apache.REEF.Tests.Functional.Driver string testFolder = DefaultRuntimeFolder + TestNumber++; CleanUp(testFolder); TestRun(DriverConfigurationsWithEvaluatorRequest(), typeof(EvaluatorRequestingDriver), 1, "EvaluatorRequestingDriver", "local", testFolder); - ValidateMessageSuccessfullyLogged("Runtime Name: Local", testFolder, 2); + ValidateMessageSuccessfullyLoggedForDriver("Runtime Name: Local", testFolder, 2); CleanUp(testFolder); }
