Repository: reef Updated Branches: refs/heads/master 6b058ba1f -> 7d62431bf
[REEF-1325] Fix PoisonTest.TestPoisonedEvaluatorStartHanlder failure in AppVeyor This change: * adds ICompletedTask handler to PoisonTest (which fixes task failues) * expands IFailedEvaluator handler to show information about associated FailedTask * refactors reading from log in ReefFunctionalTest to allow reading log file content in a test, not just in Validate functions. A follow-up JIRA will investigate why task completes after evaluator failure. JIRA: [REEF-1325](https://issues.apache.org/jira/browse/REEF-1325) Pull request: This closes #955 Project: http://git-wip-us.apache.org/repos/asf/reef/repo Commit: http://git-wip-us.apache.org/repos/asf/reef/commit/7d62431b Tree: http://git-wip-us.apache.org/repos/asf/reef/tree/7d62431b Diff: http://git-wip-us.apache.org/repos/asf/reef/diff/7d62431b Branch: refs/heads/master Commit: 7d62431bf94f87ed395dfa6cb4ff32abd52d9c67 Parents: 6b058ba Author: Mariia Mykhailova <[email protected]> Authored: Wed Apr 13 14:21:19 2016 -0700 Committer: Markus Weimer <[email protected]> Committed: Tue Apr 19 15:09:14 2016 -0700 ---------------------------------------------------------------------- .../Poison/PoisonedEventHandler.cs | 1 - .../Functional/FaultTolerant/PoisonTest.cs | 23 +++- .../Functional/ReefFunctionalTest.cs | 123 +++++++++---------- 3 files changed, 77 insertions(+), 70 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/reef/blob/7d62431b/lang/cs/Org.Apache.REEF.Common/Poison/PoisonedEventHandler.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Common/Poison/PoisonedEventHandler.cs b/lang/cs/Org.Apache.REEF.Common/Poison/PoisonedEventHandler.cs index 4ab6722..a5f07da 100644 --- a/lang/cs/Org.Apache.REEF.Common/Poison/PoisonedEventHandler.cs +++ b/lang/cs/Org.Apache.REEF.Common/Poison/PoisonedEventHandler.cs @@ -22,7 +22,6 @@ using Org.Apache.REEF.Wake.Time.Event; using Org.Apache.REEF.Wake.Time.Runtime; using System; using System.Reactive; -using System.Threading.Tasks; namespace Org.Apache.REEF.Common.Poison { http://git-wip-us.apache.org/repos/asf/reef/blob/7d62431b/lang/cs/Org.Apache.REEF.Tests/Functional/FaultTolerant/PoisonTest.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Tests/Functional/FaultTolerant/PoisonTest.cs b/lang/cs/Org.Apache.REEF.Tests/Functional/FaultTolerant/PoisonTest.cs index 72df68c..7aab1ac 100644 --- a/lang/cs/Org.Apache.REEF.Tests/Functional/FaultTolerant/PoisonTest.cs +++ b/lang/cs/Org.Apache.REEF.Tests/Functional/FaultTolerant/PoisonTest.cs @@ -29,6 +29,7 @@ using System.Threading; using Org.Apache.REEF.Common.Context; using Org.Apache.REEF.Common.Events; using Org.Apache.REEF.Common.Poison; +using Org.Apache.REEF.Driver.Task; using Org.Apache.REEF.Tang.Implementations.Tang; using Org.Apache.REEF.Tang.Implementations.Configuration; @@ -39,6 +40,7 @@ namespace Org.Apache.REEF.Tests.Functional.FaultTolerant { private static readonly Logger Logger = Logger.GetLogger(typeof(PoisonTest)); + private const string Prefix = "Poison: "; private const string FailedEvaluatorMessage = "I have succeeded in seeing a failed evaluator."; private const string TaskId = "1234567"; @@ -59,6 +61,7 @@ namespace Org.Apache.REEF.Tests.Functional.FaultTolerant .Set(DriverConfiguration.OnEvaluatorAllocated, GenericType<PoisonedEvaluatorDriver>.Class) .Set(DriverConfiguration.OnEvaluatorFailed, GenericType<PoisonedEvaluatorDriver>.Class) .Set(DriverConfiguration.OnContextActive, GenericType<PoisonedEvaluatorDriver>.Class) + .Set(DriverConfiguration.OnTaskCompleted, GenericType<PoisonedEvaluatorDriver>.Class) .Build(); } @@ -66,7 +69,8 @@ namespace Org.Apache.REEF.Tests.Functional.FaultTolerant IObserver<IDriverStarted>, IObserver<IAllocatedEvaluator>, IObserver<IActiveContext>, - IObserver<IFailedEvaluator> + IObserver<IFailedEvaluator>, + IObserver<ICompletedTask> { private readonly IEvaluatorRequestor _requestor; @@ -108,6 +112,20 @@ namespace Org.Apache.REEF.Tests.Functional.FaultTolerant public void OnNext(IFailedEvaluator value) { Logger.Log(Level.Error, FailedEvaluatorMessage); + if (value.FailedTask.Value == null) + { + // TODO[JIRA REEF-1343]: fail the test if there's no failed task + Logger.Log(Level.Error, "No failed task associated with failed evaluator"); + } + else + { + Logger.Log(Level.Error, "Failed task id '" + value.FailedTask.Value.Id + "'"); + } + } + public void OnNext(ICompletedTask value) + { + // TODO[JIRA REEF-1343]: fail the test if receive ICompletedTask after failed evaluator + Logger.Log(Level.Info, "ICompletedTask"); } public void OnError(Exception error) @@ -134,8 +152,9 @@ namespace Org.Apache.REEF.Tests.Functional.FaultTolerant public byte[] Call(byte[] memento) { - Logger.Log(Level.Verbose, "Will sleep for 2 seconds (expecting to be poisoned faster)."); + Logger.Log(Level.Info, Prefix + "Will sleep for 2 seconds (expecting to be poisoned faster)."); Thread.Sleep(2000); + Logger.Log(Level.Info, Prefix + "Task sleep finished successfully."); return null; } } http://git-wip-us.apache.org/repos/asf/reef/blob/7d62431b/lang/cs/Org.Apache.REEF.Tests/Functional/ReefFunctionalTest.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Tests/Functional/ReefFunctionalTest.cs b/lang/cs/Org.Apache.REEF.Tests/Functional/ReefFunctionalTest.cs index ca3451b..ad38652 100644 --- a/lang/cs/Org.Apache.REEF.Tests/Functional/ReefFunctionalTest.cs +++ b/lang/cs/Org.Apache.REEF.Tests/Functional/ReefFunctionalTest.cs @@ -156,37 +156,18 @@ namespace Org.Apache.REEF.Tests.Functional const string successIndication = "EXIT: ActiveContextClr2Java::Close"; const string failedTaskIndication = "Java_org_apache_reef_javabridge_NativeInterop_clrSystemFailedTaskHandlerOnNext"; const string failedEvaluatorIndication = "Java_org_apache_reef_javabridge_NativeInterop_clrSystemFailedEvaluatorHandlerOnNext"; - string[] lines = null; - for (int i = 0; i < 60; i++) - { - try - { - lines = File.ReadAllLines(GetLogFile(DriverStdout, "driver", testFolder)); - break; - } - catch (Exception) - { - Thread.Sleep(SleepTime); - } - } - - if (lines != null) - { - Logger.Log(Level.Verbose, "Lines read from log file : " + lines.Count()); - string[] successIndicators = lines.Where(s => s.Contains(successIndication)).ToArray(); - string[] failedTaskIndicators = lines.Where(s => s.Contains(failedTaskIndication)).ToArray(); - string[] failedEvaluatorIndicators = lines.Where(s => s.Contains(failedEvaluatorIndication)).ToArray(); - Assert.True(numberOfContextsToClose == successIndicators.Length, - "Expected number of contexts to close (" + numberOfContextsToClose + ") differs from actual number of success indicators (" + successIndicators.Length + ")"); - Assert.True(numberOfTasksToFail == failedTaskIndicators.Length, - "Expected number of tasks to fail (" + numberOfTasksToFail + ") differs from actual number of failed task indicators (" + failedTaskIndicators.Length + ")"); - Assert.True(numberOfEvaluatorsToFail == failedEvaluatorIndicators.Length, - "Expected number of evaluators to fail (" + numberOfEvaluatorsToFail + ") differs from actual number of failed evaluator indicators (" + failedEvaluatorIndicators.Length + ")"); - } - else - { - Assert.True(false, "Cannot read from log file " + DriverStdout); - } + string[] lines = ReadLogFile(DriverStdout, "driver", testFolder); + + Logger.Log(Level.Verbose, "Lines read from log file : " + lines.Count()); + string[] successIndicators = lines.Where(s => s.Contains(successIndication)).ToArray(); + string[] failedTaskIndicators = lines.Where(s => s.Contains(failedTaskIndication)).ToArray(); + string[] failedEvaluatorIndicators = lines.Where(s => s.Contains(failedEvaluatorIndication)).ToArray(); + Assert.True(numberOfContextsToClose == successIndicators.Length, + "Expected number of contexts to close (" + numberOfContextsToClose + ") differs from actual number of success indicators (" + successIndicators.Length + ")"); + Assert.True(numberOfTasksToFail == failedTaskIndicators.Length, + "Expected number of tasks to fail (" + numberOfTasksToFail + ") differs from actual number of failed task indicators (" + failedTaskIndicators.Length + ")"); + Assert.True(numberOfEvaluatorsToFail == failedEvaluatorIndicators.Length, + "Expected number of evaluators to fail (" + numberOfEvaluatorsToFail + ") differs from actual number of failed evaluator indicators (" + failedEvaluatorIndicators.Length + ")"); } /// <summary> @@ -209,46 +190,25 @@ namespace Org.Apache.REEF.Tests.Functional /// </summary> protected void ValidateMessageSuccessfullyLogged(IList<string> messages, string subfolder, string fileName, string testFolder, int numberOfOccurrences = 1) { - string[] lines = null; - for (int i = 0; i < 60; i++) + string[] lines = ReadLogFile(fileName, subfolder, testFolder); + foreach (string message in messages) { - try + string[] successIndicators = lines.Where(s => s.Contains(message)).ToArray(); + if (numberOfOccurrences > 0) { - lines = File.ReadAllLines(GetLogFile(fileName, subfolder, testFolder)); - break; + Assert.True(numberOfOccurrences == successIndicators.Count(), + "Expected number of message occurrences " + numberOfOccurrences + " differs from actual " + successIndicators.Count()); } - catch (Exception e) + else if (numberOfOccurrences == 0) { - Logger.Log(Level.Verbose, e.ToString()); - Thread.Sleep(SleepTime); + Assert.True(0 == successIndicators.Count(), + "Message not expected to occur but occurs " + successIndicators.Count() + " times"); } - } - - if (lines != null) - { - foreach (string message in messages) + else { - string[] successIndicators = lines.Where(s => s.Contains(message)).ToArray(); - if (numberOfOccurrences > 0) - { - Assert.True(numberOfOccurrences == successIndicators.Count(), - "Expected number of message occurrences " + numberOfOccurrences + " differs from actual " + successIndicators.Count()); - } - else if (numberOfOccurrences == 0) - { - Assert.True(0 == successIndicators.Count(), - "Message not expected to occur but occurs " + successIndicators.Count() + " times"); - } - else - { - Assert.True(successIndicators.Count() > 0, "Message expected to occur, but did not."); - } + Assert.True(successIndicators.Count() > 0, "Message expected to occur, but did not."); } } - else - { - Assert.True(false, "Cannot read from log file " + fileName); - } } protected void PeriodicUploadLog(object source, ElapsedEventArgs e) @@ -263,10 +223,39 @@ namespace Org.Apache.REEF.Tests.Functional } } - protected string GetLogFile(string logFileName, string subfolder = "driver", string testFolder = DefaultRuntimeFolder) + internal string[] ReadLogFile(string logFileName, string subfolder = "driver", string testFolder = DefaultRuntimeFolder) + { + string fileName = string.Empty; + string[] lines = null; + for (int i = 0; i < 60; i++) + { + try + { + fileName = GetLogFileName(logFileName, subfolder, testFolder); + lines = File.ReadAllLines(fileName); + break; + } + catch (Exception e) + { + if (i == 59) + { + // log only last exception before failure + Logger.Log(Level.Verbose, e.ToString()); + } + if (i < 59) + { + Thread.Sleep(SleepTime); + } + } + } + Assert.True(lines != null, "Cannot read from log file " + fileName); + return lines; + } + + protected string GetLogFileName(string logFileName, string subfolder = "driver", string testFolder = DefaultRuntimeFolder) { string driverContainerDirectory = Directory.GetDirectories(Path.Combine(Directory.GetCurrentDirectory(), testFolder), subfolder, SearchOption.AllDirectories).SingleOrDefault(); - Logger.Log(Level.Verbose, "GetLogFile, driverContainerDirectory:" + driverContainerDirectory); + Logger.Log(Level.Verbose, "GetLogFileName, driverContainerDirectory:" + driverContainerDirectory); if (string.IsNullOrWhiteSpace(driverContainerDirectory)) { @@ -282,8 +271,8 @@ namespace Org.Apache.REEF.Tests.Functional private void UploadDriverLog() { - string driverStdout = GetLogFile(DriverStdout); - string driverStderr = GetLogFile(DriverStderr); + string driverStdout = GetLogFileName(DriverStdout); + string driverStderr = GetLogFileName(DriverStderr); CloudStorageAccount storageAccount = CloudStorageAccount.Parse(GetStorageConnectionString()); CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); CloudBlobContainer container = blobClient.GetContainerReference(DateTime.Now.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture));
