Repository: reef Updated Branches: refs/heads/master 09d56bcfb -> 415bb9eb3
[REEF-1628] Clean up temporary files after REEF test runs This change: * cleans up KMeansInput-* files generated by TestKMeans. * cleans up temp folders and files created by REEF.IO.Tests. * cleans up IMRUBroadcastReduceWithFilePartitionDataSetTest. JIRA: [REEF-1628](https://issues.apache.org/jira/browse/REEF-1628) Pull request: This closes #1206 Project: http://git-wip-us.apache.org/repos/asf/reef/repo Commit: http://git-wip-us.apache.org/repos/asf/reef/commit/415bb9eb Tree: http://git-wip-us.apache.org/repos/asf/reef/tree/415bb9eb Diff: http://git-wip-us.apache.org/repos/asf/reef/diff/415bb9eb Branch: refs/heads/master Commit: 415bb9eb3dabf93153a20c462c80eb71b679e342 Parents: 09d56bc Author: Mariia Mykhailova <[email protected]> Authored: Thu Dec 15 11:51:27 2016 -0800 Committer: Julia Wang <[email protected]> Committed: Tue Dec 20 14:26:36 2016 -0800 ---------------------------------------------------------------------- .../TestFilePartitionInputDataSet.cs | 18 +++++++++--- .../TestTempFolderCreator.cs | 15 ++++++++-- ...oadcastReduceWithFilePartitionDataSetTest.cs | 29 ++++++++++++-------- .../Functional/ML/KMeans/TestKMeans.cs | 16 +++++++++++ 4 files changed, 59 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/reef/blob/415bb9eb/lang/cs/Org.Apache.REEF.IO.Tests/TestFilePartitionInputDataSet.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.IO.Tests/TestFilePartitionInputDataSet.cs b/lang/cs/Org.Apache.REEF.IO.Tests/TestFilePartitionInputDataSet.cs index 001c322..0695f9f 100644 --- a/lang/cs/Org.Apache.REEF.IO.Tests/TestFilePartitionInputDataSet.cs +++ b/lang/cs/Org.Apache.REEF.IO.Tests/TestFilePartitionInputDataSet.cs @@ -35,7 +35,7 @@ namespace Org.Apache.REEF.IO.Tests /// <summary> /// Tests for Org.Apache.REEF.IO.PartitionedData.FileSystem. /// </summary> - public class TestFilePartitionInputDataSet + public class TestFilePartitionInputDataSet : IDisposable { private static readonly Logger Logger = Logger.GetLogger(typeof(TestFilePartitionInputDataSet)); @@ -266,15 +266,25 @@ namespace Org.Apache.REEF.IO.Tests } } Assert.Equal(count, 5); + + // clean up created folders + Directory.Delete("./test2/", true); } - private void MakeLocalTestFile(string filePath, byte[] bytes) + public void Dispose() { - if (File.Exists(filePath)) + if (File.Exists(sourceFilePath1)) + { + File.Delete(sourceFilePath1); + } + if (File.Exists(sourceFilePath2)) { - File.Delete(filePath); + File.Delete(sourceFilePath2); } + } + private void MakeLocalTestFile(string filePath, byte[] bytes) + { using (var s = File.Create(filePath)) { foreach (var b in bytes) http://git-wip-us.apache.org/repos/asf/reef/blob/415bb9eb/lang/cs/Org.Apache.REEF.IO.Tests/TestTempFolderCreator.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.IO.Tests/TestTempFolderCreator.cs b/lang/cs/Org.Apache.REEF.IO.Tests/TestTempFolderCreator.cs index d3be4fd..fdd0e25 100644 --- a/lang/cs/Org.Apache.REEF.IO.Tests/TestTempFolderCreator.cs +++ b/lang/cs/Org.Apache.REEF.IO.Tests/TestTempFolderCreator.cs @@ -37,6 +37,9 @@ namespace Org.Apache.REEF.IO.Tests var f2 = (string)i.GetNamedInstance(typeof(TempFileFolder)); var f = Path.GetFullPath(f2); Assert.True(f1.StartsWith(f)); + + // clean up created folders + Directory.Delete(f, true); } /// <summary> @@ -46,7 +49,7 @@ namespace Org.Apache.REEF.IO.Tests public void TestTempFileFolerParameter() { var b = TempFileConfigurationModule.ConfigurationModule - .Set(TempFileConfigurationModule.TempFileFolerParameter, @".\test1\abc\") + .Set(TempFileConfigurationModule.TempFileFolerParameter, "./test1/abc/") .Build(); var i = TangFactory.GetTang().NewInjector(b); var tempFileCreator = i.GetInstance<ITempFileCreator>(); @@ -54,6 +57,9 @@ namespace Org.Apache.REEF.IO.Tests var f2 = (string)i.GetNamedInstance(typeof(TempFileFolder)); var f = Path.GetFullPath(f2); Assert.True(f1.StartsWith(f)); + + // clean up created folders + Directory.Delete("./test1/", true); } /// <summary> @@ -63,15 +69,18 @@ namespace Org.Apache.REEF.IO.Tests public void TestCreateTempFileFoler() { var b = TempFileConfigurationModule.ConfigurationModule - .Set(TempFileConfigurationModule.TempFileFolerParameter, @"./test1/abc/") + .Set(TempFileConfigurationModule.TempFileFolerParameter, @"./test3/abc/") .Build(); var i = TangFactory.GetTang().NewInjector(b); var tempFileCreator = i.GetInstance<ITempFileCreator>(); var f1 = tempFileCreator.CreateTempDirectory("ddd\\fff"); var f3 = tempFileCreator.CreateTempDirectory("ddd\\fff", "bbb"); - var f2 = Path.GetFullPath(@"./test1/abc/" + "ddd\\fff"); + var f2 = Path.GetFullPath(@"./test3/abc/" + "ddd\\fff"); Assert.True(f1.StartsWith(f2)); Assert.True(f3.EndsWith("bbb")); + + // clean up created folders + Directory.Delete("./test3/", true); } } } http://git-wip-us.apache.org/repos/asf/reef/blob/415bb9eb/lang/cs/Org.Apache.REEF.Tests/Functional/IMRU/IMRUBroadcastReduceWithFilePartitionDataSetTest.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Tests/Functional/IMRU/IMRUBroadcastReduceWithFilePartitionDataSetTest.cs b/lang/cs/Org.Apache.REEF.Tests/Functional/IMRU/IMRUBroadcastReduceWithFilePartitionDataSetTest.cs index c68517c..4f5d9f5 100644 --- a/lang/cs/Org.Apache.REEF.Tests/Functional/IMRU/IMRUBroadcastReduceWithFilePartitionDataSetTest.cs +++ b/lang/cs/Org.Apache.REEF.Tests/Functional/IMRU/IMRUBroadcastReduceWithFilePartitionDataSetTest.cs @@ -49,7 +49,9 @@ namespace Org.Apache.REEF.Tests.Functional.IMRU /// <summary> /// Defines the number of data in input data bytes in the test /// </summary> - protected const int DataCount = 3; + private const int DataCount = 3; + + private const string tempFileName = "REEF.TestLocalFileSystem.tmp"; /// <summary> /// This test tests DataLoadingContext with FilePartitionDataSet @@ -79,6 +81,15 @@ namespace Org.Apache.REEF.Tests.Functional.IMRU testFolder); ValidateSuccessForLocalRuntime(numTasks, 0, 0, testFolder, 100); CleanUp(testFolder); + + for (var i = 0; i < numTasks; i++) + { + var filePath = Path.Combine(Path.GetTempPath(), tempFileName + i); + if (File.Exists(filePath)) + { + File.Delete(filePath); + } + } } /// <summary> @@ -140,7 +151,7 @@ namespace Org.Apache.REEF.Tests.Functional.IMRU PartitionDatasetConfig(); } - protected void PartitionDatasetConfig() + private void PartitionDatasetConfig() { int count = 0; int numberOfTasks = 4; @@ -190,7 +201,7 @@ namespace Org.Apache.REEF.Tests.Functional.IMRU /// It then iterates the data and verifies the data count. /// </summary> /// <typeparam name="T"></typeparam> - internal sealed class TestSenderMapFunction<T> : IMapFunction<int[], int[]> + private sealed class TestSenderMapFunction<T> : IMapFunction<int[], int[]> { private int _iterations; @@ -277,7 +288,6 @@ namespace Org.Apache.REEF.Tests.Functional.IMRU protected static ConfigurationModule CreateDatasetConfigurationModule(int numberofMappers) { - const string tempFileName = "REEF.TestLocalFileSystem.tmp"; string sourceFilePath = Path.Combine(Path.GetTempPath(), tempFileName); var cm = FileSystemInputPartitionConfiguration<IEnumerable<Row>>.ConfigurationModule; @@ -295,7 +305,7 @@ namespace Org.Apache.REEF.Tests.Functional.IMRU /// </summary> /// <param name="dataNumber"></param> /// <returns></returns> - protected static byte[] CreateTestData(int dataNumber) + private static byte[] CreateTestData(int dataNumber) { var bytes = new byte[dataNumber]; for (int i = 0; i < dataNumber; i++) @@ -310,13 +320,8 @@ namespace Org.Apache.REEF.Tests.Functional.IMRU /// </summary> /// <param name="filePath"></param> /// <param name="bytes"></param> - protected static void MakeLocalTestFile(string filePath, byte[] bytes) + private static void MakeLocalTestFile(string filePath, byte[] bytes) { - if (File.Exists(filePath)) - { - File.Delete(filePath); - } - using (var s = File.Create(filePath)) { foreach (var b in bytes) @@ -329,7 +334,7 @@ namespace Org.Apache.REEF.Tests.Functional.IMRU /// <summary> /// Test DeSerializer /// </summary> - protected class RowSerializer : IFileDeSerializer<IEnumerable<Row>> + private class RowSerializer : IFileDeSerializer<IEnumerable<Row>> { [Inject] private RowSerializer() http://git-wip-us.apache.org/repos/asf/reef/blob/415bb9eb/lang/cs/Org.Apache.REEF.Tests/Functional/ML/KMeans/TestKMeans.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Tests/Functional/ML/KMeans/TestKMeans.cs b/lang/cs/Org.Apache.REEF.Tests/Functional/ML/KMeans/TestKMeans.cs index 4db2b3b..a1cc91c 100644 --- a/lang/cs/Org.Apache.REEF.Tests/Functional/ML/KMeans/TestKMeans.cs +++ b/lang/cs/Org.Apache.REEF.Tests/Functional/ML/KMeans/TestKMeans.cs @@ -101,6 +101,14 @@ namespace Org.Apache.REEF.Tests.Functional.ML.KMeans { // do not fail if clean up is unsuccessful } + try + { + File.Delete(dataFilePath); + } + catch (Exception) + { + // do not fail if clean up is unsuccessful + } } [Fact] @@ -117,6 +125,14 @@ namespace Org.Apache.REEF.Tests.Functional.ML.KMeans TestRun(DriverConfiguration(dataFilePath), typeof(KMeansDriverHandlers), Partitions + 1, "KMeansDriverHandlers", "local", testFolder); ValidateSuccessForLocalRuntime(Partitions + 1, testFolder: testFolder); CleanUp(testFolder); + try + { + File.Delete(dataFilePath); + } + catch (Exception) + { + // do not fail if clean up is unsuccessful + } } [Fact(Skip = "Requires Yarn Single Node")]
