[CARBONDATA-1114][Tests] Fix bugs in tests in windows env Fix bugs in tests that will cause failure under windows env
This closes #1994 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/58c17539 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/58c17539 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/58c17539 Branch: refs/heads/carbonstore-rebase5 Commit: 58c17539b468e6ae089dfc51fbf3af85698047fc Parents: 2414fb0 Author: xuchuanyin <xuchuan...@hust.edu.cn> Authored: Sat Feb 24 21:18:17 2018 +0800 Committer: Jacky Li <jacky.li...@qq.com> Committed: Wed Feb 28 22:05:23 2018 +0800 ---------------------------------------------------------------------- .../carbondata/core/locks/LocalFileLock.java | 30 +++++++++++--------- .../store/impl/DFSFileReaderImplUnitTest.java | 11 ++++--- .../store/impl/FileFactoryImplUnitTest.java | 28 +++++++++++++----- .../filesystem/HDFSCarbonFileTest.java | 3 +- .../filesystem/LocalCarbonFileTest.java | 20 +++++++++---- .../BooleanDataTypesInsertTest.scala | 5 ++-- .../carbondata/lcm/locks/LocalFileLockTest.java | 2 +- .../loading/csvinput/CSVInputFormatTest.java | 1 + 8 files changed, 64 insertions(+), 36 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/58c17539/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java index 75ea074..cb80877 100644 --- a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java +++ b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java @@ -17,17 +17,20 @@ package org.apache.carbondata.core.locks; -import java.io.FileOutputStream; import java.io.IOException; import java.nio.channels.FileChannel; import java.nio.channels.FileLock; import java.nio.channels.OverlappingFileLockException; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.datastore.filesystem.CarbonFile; import org.apache.carbondata.core.datastore.impl.FileFactory; import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; +import org.apache.carbondata.core.util.CarbonUtil; /** * This class handles the file locking in the local file system. @@ -40,11 +43,6 @@ public class LocalFileLock extends AbstractCarbonLock { private String location; /** - * fileOutputStream of the local lock file - */ - private FileOutputStream fileOutputStream; - - /** * channel is the FileChannel of the lock file. */ private FileChannel channel; @@ -104,8 +102,8 @@ public class LocalFileLock extends AbstractCarbonLock { FileFactory.createNewLockFile(lockFilePath, FileFactory.getFileType(location)); } - fileOutputStream = new FileOutputStream(lockFilePath); - channel = fileOutputStream.getChannel(); + channel = FileChannel.open(Paths.get(lockFilePath), StandardOpenOption.WRITE, + StandardOpenOption.APPEND); try { fileLock = channel.tryLock(); } catch (OverlappingFileLockException e) { @@ -137,11 +135,17 @@ public class LocalFileLock extends AbstractCarbonLock { } catch (IOException e) { status = false; } finally { - if (null != fileOutputStream) { - try { - fileOutputStream.close(); - } catch (IOException e) { - LOGGER.error(e.getMessage()); + CarbonUtil.closeStreams(channel); + + // deleting the lock file after releasing the lock. + if (null != lockFilePath) { + CarbonFile lockFile = FileFactory.getCarbonFile(lockFilePath, + FileFactory.getFileType(lockFilePath)); + if (!lockFile.exists() || lockFile.delete()) { + LOGGER.info("Successfully deleted the lock file " + lockFilePath); + } else { + LOGGER.error("Not able to delete the lock file " + lockFilePath); + status = false; } } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/58c17539/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java index da61a94..30144c1 100644 --- a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java +++ b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java @@ -38,7 +38,6 @@ import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertThat; public class DFSFileReaderImplUnitTest { - private static DFSFileReaderImpl dfsFileHolder; private static String fileName; private static String fileNameWithEmptyContent; @@ -50,10 +49,8 @@ public class DFSFileReaderImplUnitTest { file = new File("Test.carbondata"); fileWithEmptyContent = new File("TestEXception.carbondata"); - if (!file.exists()) try { - file.createNewFile(); - } catch (IOException e) { - e.printStackTrace(); + if (file.exists()) { + file.delete(); } if (!fileWithEmptyContent.exists()) try { fileWithEmptyContent.createNewFile(); @@ -61,10 +58,12 @@ public class DFSFileReaderImplUnitTest { e.printStackTrace(); } try { - FileOutputStream of = new FileOutputStream(file, true); + FileOutputStream of = new FileOutputStream(file, false); BufferedWriter br = new BufferedWriter(new OutputStreamWriter(of, "UTF-8")); br.write("Hello World"); br.close(); + of.flush(); + of.close(); } catch (Exception e) { e.getMessage(); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/58c17539/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileFactoryImplUnitTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileFactoryImplUnitTest.java b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileFactoryImplUnitTest.java index 65590d6..0e7d1c9 100644 --- a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileFactoryImplUnitTest.java +++ b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileFactoryImplUnitTest.java @@ -17,6 +17,7 @@ package org.apache.carbondata.core.carbon.datastorage.filesystem.store.impl; +import java.io.DataOutputStream; import java.io.File; import java.io.FileWriter; import java.io.IOException; @@ -43,6 +44,10 @@ public class FileFactoryImplUnitTest { @AfterClass public static void tearDown() { + cleanUp(); + } + + private static void cleanUp() { File file = new File(filePath); if (file.exists()) { file.delete(); @@ -83,17 +88,17 @@ public class FileFactoryImplUnitTest { } @Test public void testCreateNewFileWithDefaultFileType() throws IOException { - tearDown(); + cleanUp(); assertTrue(FileFactory.createNewFile(filePath, FileFactory.FileType.LOCAL)); } @Test public void testCreateNewLockFileWithDefaultFileType() throws IOException { - tearDown(); + cleanUp(); assertTrue(FileFactory.createNewLockFile(filePath, FileFactory.FileType.LOCAL)); } @Test public void testCreateNewLockFileWithViewFsFileType() throws IOException { - tearDown(); + cleanUp(); assertTrue(FileFactory.createNewLockFile(filePath, FileFactory.FileType.VIEWFS)); } @@ -129,20 +134,29 @@ public class FileFactoryImplUnitTest { assertTrue(FileFactory.mkdirs(filePath, FileFactory.FileType.VIEWFS)); } - @Test public void testGetDataOutputStreamUsingAppendeForException() { + @Test public void testGetDataOutputStreamUsingAppendeForException() throws IOException { + DataOutputStream outputStream = null; try { - FileFactory.getDataOutputStreamUsingAppend(filePath, FileFactory.FileType.VIEWFS); + outputStream = FileFactory.getDataOutputStreamUsingAppend(filePath, FileFactory.FileType.VIEWFS); } catch (Exception exception) { assertEquals("Not supported", exception.getMessage()); + } finally { + if (null != outputStream) { + outputStream.close(); + } } } @Test public void getDataOutputStreamForVIEWFSType() throws IOException { - assertNotNull(FileFactory.getDataOutputStream(filePath, FileFactory.FileType.VIEWFS)); + DataOutputStream outputStream = FileFactory.getDataOutputStream(filePath, FileFactory.FileType.VIEWFS); + assertNotNull(outputStream); + outputStream.close(); } @Test public void getDataOutputStreamForLocalType() throws IOException { - assertNotNull(FileFactory.getDataOutputStream(filePath, FileFactory.FileType.LOCAL)); + DataOutputStream outputStream = FileFactory.getDataOutputStream(filePath, FileFactory.FileType.LOCAL); + assertNotNull(outputStream); + outputStream.close(); } @Test public void testGetCarbonFile() throws IOException { http://git-wip-us.apache.org/repos/asf/carbondata/blob/58c17539/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java index 4018123..42d4afa 100644 --- a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java +++ b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java @@ -58,7 +58,7 @@ public class HDFSCarbonFileTest { @BeforeClass static public void setUp() throws IOException { Configuration config = new Configuration(); -//adding local hadoop configuration + // adding local hadoop configuration config.addResource(new Path("core-site.xml")); config.addResource(new Path("hdfs-site.xml")); fileName = "Test.carbondata"; //this path is HDFS path @@ -75,6 +75,7 @@ public class HDFSCarbonFileTest { BufferedWriter br = new BufferedWriter(new OutputStreamWriter(os, "UTF-8")); br.write("Hello World"); br.close(); + os.close(); fs.close(); fileStatus = new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName)); http://git-wip-us.apache.org/repos/asf/carbondata/blob/58c17539/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFileTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFileTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFileTest.java index 96ef106..14f9fe2 100644 --- a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFileTest.java +++ b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFileTest.java @@ -29,6 +29,7 @@ import sun.nio.ch.FileChannelImpl; import java.io.*; import java.nio.channels.ReadableByteChannel; import java.util.Objects; +import java.util.UUID; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; @@ -44,8 +45,8 @@ public class LocalCarbonFileTest { @BeforeClass static public void setUp() { - file = new File("Test.carbondata"); - dir = new File("Testdir.carbondata"); + file = new File("TestLocalCarbonFile"); + dir = new File("TestLocalCarbonDir"); if (!file.exists()) try { file.createNewFile(); @@ -60,6 +61,7 @@ public class LocalCarbonFileTest { byte[] bytes = "core java api".getBytes(); oFile.write(bytes); + oFile.close(); } catch (FileNotFoundException e) { e.printStackTrace(); localCarbonFile = new LocalCarbonFile(file); @@ -121,8 +123,9 @@ public class LocalCarbonFileTest { @Test public void testRenameForce() { localCarbonFile = new LocalCarbonFile(file); - assertTrue(localCarbonFile.renameForce("Testdb.carbon")); - File file1 = new File("Testdb.carbon"); + String destFile = "TestRename" + UUID.randomUUID().toString(); + assertTrue(localCarbonFile.renameForce(destFile)); + File file1 = new File(destFile); if (file1.exists()) { file1.delete(); } @@ -131,7 +134,12 @@ public class LocalCarbonFileTest { @Test public void testRenameTo() { localCarbonFile = new LocalCarbonFile(file); - assertTrue(!localCarbonFile.renameTo("Testdb.carbon")); + String destFile = "TestRename" + UUID.randomUUID().toString(); + assertTrue(!localCarbonFile.renameTo(destFile)); + File file1 = new File(destFile); + if (file1.exists()) { + file1.delete(); + } } @Test @@ -463,6 +471,6 @@ public class LocalCarbonFileTest { localCarbonFile = new LocalCarbonFile("demo.txt"); - assertEquals(localCarbonFile.renameForce("Test.carbondata"), true); + assertEquals(localCarbonFile.renameForce("renameToFile"), true); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/58c17539/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesInsertTest.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesInsertTest.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesInsertTest.scala index f8cfa6b..45edd3d 100644 --- a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesInsertTest.scala +++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesInsertTest.scala @@ -23,6 +23,7 @@ import org.apache.spark.sql.test.util.QueryTest import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach} import org.apache.carbondata.core.constants.CarbonCommonConstants +import org.apache.carbondata.core.datastore.impl.FileFactory import org.apache.carbondata.core.util.CarbonProperties class BooleanDataTypesInsertTest extends QueryTest with BeforeAndAfterEach with BeforeAndAfterAll { @@ -555,7 +556,7 @@ class BooleanDataTypesInsertTest extends QueryTest with BeforeAndAfterEach with sql( s""" - | LOAD DATA LOCAL INPATH '${storeLocation}' + | LOAD DATA LOCAL INPATH '${FileFactory.getUpdatedFilePath(storeLocation)}' | INTO TABLE hive_table """.stripMargin) @@ -923,7 +924,7 @@ class BooleanDataTypesInsertTest extends QueryTest with BeforeAndAfterEach with sql( s""" - | LOAD DATA LOCAL INPATH '${storeLocation}' + | LOAD DATA LOCAL INPATH '${FileFactory.getUpdatedFilePath(storeLocation)}' | INTO TABLE hive_table """.stripMargin) http://git-wip-us.apache.org/repos/asf/carbondata/blob/58c17539/processing/src/test/java/org/apache/carbondata/lcm/locks/LocalFileLockTest.java ---------------------------------------------------------------------- diff --git a/processing/src/test/java/org/apache/carbondata/lcm/locks/LocalFileLockTest.java b/processing/src/test/java/org/apache/carbondata/lcm/locks/LocalFileLockTest.java index 812db5c..8d5f3d4 100644 --- a/processing/src/test/java/org/apache/carbondata/lcm/locks/LocalFileLockTest.java +++ b/processing/src/test/java/org/apache/carbondata/lcm/locks/LocalFileLockTest.java @@ -65,7 +65,7 @@ public class LocalFileLockTest { Assert.assertTrue(localLock1.unlock()); Assert.assertTrue(localLock2.lock()); - + Assert.assertTrue(localLock2.unlock()); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/58c17539/processing/src/test/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormatTest.java ---------------------------------------------------------------------- diff --git a/processing/src/test/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormatTest.java b/processing/src/test/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormatTest.java index 14c680e..9a9c662 100644 --- a/processing/src/test/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormatTest.java +++ b/processing/src/test/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormatTest.java @@ -147,6 +147,7 @@ public class CSVInputFormatTest extends TestCase { FileOutputFormat.setOutputPath(job, new Path(output.getCanonicalPath())); Assert.assertTrue(job.waitForCompletion(true)); + deleteOutput(output); } private void prepareConf(Configuration conf) {