[CARBONDATA-1114][Tests] Fix bugs in tests in windows env

Fix bugs in tests that will cause failure under windows env

This closes #1994


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/8a9dd8b2
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/8a9dd8b2
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/8a9dd8b2

Branch: refs/heads/carbonstore-rebase5
Commit: 8a9dd8b21907096ded81fb58e152eecc3f629bb8
Parents: b1c9628
Author: xuchuanyin <xuchuan...@hust.edu.cn>
Authored: Sat Feb 24 21:18:17 2018 +0800
Committer: Jacky Li <jacky.li...@qq.com>
Committed: Fri Mar 2 15:52:37 2018 +0800

----------------------------------------------------------------------
 .../carbondata/core/locks/LocalFileLock.java    | 30 +++++++++++---------
 .../store/impl/DFSFileReaderImplUnitTest.java   | 11 ++++---
 .../store/impl/FileFactoryImplUnitTest.java     | 28 +++++++++++++-----
 .../filesystem/HDFSCarbonFileTest.java          |  3 +-
 .../filesystem/LocalCarbonFileTest.java         | 20 +++++++++----
 .../BooleanDataTypesInsertTest.scala            |  5 ++--
 .../carbondata/lcm/locks/LocalFileLockTest.java |  2 +-
 .../loading/csvinput/CSVInputFormatTest.java    |  1 +
 8 files changed, 64 insertions(+), 36 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/8a9dd8b2/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
----------------------------------------------------------------------
diff --git 
a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java 
b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
index 75ea074..cb80877 100644
--- a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
+++ b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
@@ -17,17 +17,20 @@
 
 package org.apache.carbondata.core.locks;
 
-import java.io.FileOutputStream;
 import java.io.IOException;
 import java.nio.channels.FileChannel;
 import java.nio.channels.FileLock;
 import java.nio.channels.OverlappingFileLockException;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
 
 import org.apache.carbondata.common.logging.LogService;
 import org.apache.carbondata.common.logging.LogServiceFactory;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
 import org.apache.carbondata.core.datastore.impl.FileFactory;
 import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.util.CarbonUtil;
 
 /**
  * This class handles the file locking in the local file system.
@@ -40,11 +43,6 @@ public class LocalFileLock extends AbstractCarbonLock {
   private String location;
 
   /**
-   * fileOutputStream of the local lock file
-   */
-  private FileOutputStream fileOutputStream;
-
-  /**
    * channel is the FileChannel of the lock file.
    */
   private FileChannel channel;
@@ -104,8 +102,8 @@ public class LocalFileLock extends AbstractCarbonLock {
         FileFactory.createNewLockFile(lockFilePath, 
FileFactory.getFileType(location));
       }
 
-      fileOutputStream = new FileOutputStream(lockFilePath);
-      channel = fileOutputStream.getChannel();
+      channel = FileChannel.open(Paths.get(lockFilePath), 
StandardOpenOption.WRITE,
+          StandardOpenOption.APPEND);
       try {
         fileLock = channel.tryLock();
       } catch (OverlappingFileLockException e) {
@@ -137,11 +135,17 @@ public class LocalFileLock extends AbstractCarbonLock {
     } catch (IOException e) {
       status = false;
     } finally {
-      if (null != fileOutputStream) {
-        try {
-          fileOutputStream.close();
-        } catch (IOException e) {
-          LOGGER.error(e.getMessage());
+      CarbonUtil.closeStreams(channel);
+
+      // deleting the lock file after releasing the lock.
+      if (null != lockFilePath) {
+        CarbonFile lockFile = FileFactory.getCarbonFile(lockFilePath,
+            FileFactory.getFileType(lockFilePath));
+        if (!lockFile.exists() || lockFile.delete()) {
+          LOGGER.info("Successfully deleted the lock file " + lockFilePath);
+        } else {
+          LOGGER.error("Not able to delete the lock file " + lockFilePath);
+          status = false;
         }
       }
     }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/8a9dd8b2/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
----------------------------------------------------------------------
diff --git 
a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
 
b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
index da61a94..30144c1 100644
--- 
a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
+++ 
b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
@@ -38,7 +38,6 @@ import static org.hamcrest.CoreMatchers.is;
 import static org.junit.Assert.assertThat;
 
 public class DFSFileReaderImplUnitTest {
-
   private static DFSFileReaderImpl dfsFileHolder;
   private static String fileName;
   private static String fileNameWithEmptyContent;
@@ -50,10 +49,8 @@ public class DFSFileReaderImplUnitTest {
     file = new File("Test.carbondata");
     fileWithEmptyContent = new File("TestEXception.carbondata");
 
-    if (!file.exists()) try {
-      file.createNewFile();
-    } catch (IOException e) {
-      e.printStackTrace();
+    if (file.exists()) {
+      file.delete();
     }
     if (!fileWithEmptyContent.exists()) try {
       fileWithEmptyContent.createNewFile();
@@ -61,10 +58,12 @@ public class DFSFileReaderImplUnitTest {
       e.printStackTrace();
     }
     try {
-      FileOutputStream of = new FileOutputStream(file, true);
+      FileOutputStream of = new FileOutputStream(file, false);
       BufferedWriter br = new BufferedWriter(new OutputStreamWriter(of, 
"UTF-8"));
       br.write("Hello World");
       br.close();
+      of.flush();
+      of.close();
     } catch (Exception e) {
       e.getMessage();
     }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/8a9dd8b2/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileFactoryImplUnitTest.java
----------------------------------------------------------------------
diff --git 
a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileFactoryImplUnitTest.java
 
b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileFactoryImplUnitTest.java
index 65590d6..0e7d1c9 100644
--- 
a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileFactoryImplUnitTest.java
+++ 
b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileFactoryImplUnitTest.java
@@ -17,6 +17,7 @@
 
 package org.apache.carbondata.core.carbon.datastorage.filesystem.store.impl;
 
+import java.io.DataOutputStream;
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
@@ -43,6 +44,10 @@ public class FileFactoryImplUnitTest {
 
   @AfterClass
   public static void tearDown() {
+    cleanUp();
+  }
+
+  private static void cleanUp() {
     File file = new File(filePath);
     if (file.exists()) {
       file.delete();
@@ -83,17 +88,17 @@ public class FileFactoryImplUnitTest {
   }
 
   @Test public void testCreateNewFileWithDefaultFileType() throws IOException {
-    tearDown();
+    cleanUp();
     assertTrue(FileFactory.createNewFile(filePath, 
FileFactory.FileType.LOCAL));
   }
 
   @Test public void testCreateNewLockFileWithDefaultFileType() throws 
IOException {
-    tearDown();
+    cleanUp();
     assertTrue(FileFactory.createNewLockFile(filePath, 
FileFactory.FileType.LOCAL));
   }
 
   @Test public void testCreateNewLockFileWithViewFsFileType() throws 
IOException {
-    tearDown();
+    cleanUp();
     assertTrue(FileFactory.createNewLockFile(filePath, 
FileFactory.FileType.VIEWFS));
   }
 
@@ -129,20 +134,29 @@ public class FileFactoryImplUnitTest {
     assertTrue(FileFactory.mkdirs(filePath, FileFactory.FileType.VIEWFS));
   }
 
-  @Test public void testGetDataOutputStreamUsingAppendeForException() {
+  @Test public void testGetDataOutputStreamUsingAppendeForException() throws 
IOException {
+    DataOutputStream outputStream = null;
     try {
-      FileFactory.getDataOutputStreamUsingAppend(filePath, 
FileFactory.FileType.VIEWFS);
+      outputStream = FileFactory.getDataOutputStreamUsingAppend(filePath, 
FileFactory.FileType.VIEWFS);
     } catch (Exception exception) {
       assertEquals("Not supported", exception.getMessage());
+    } finally {
+      if (null != outputStream) {
+        outputStream.close();
+      }
     }
   }
 
   @Test public void getDataOutputStreamForVIEWFSType() throws IOException {
-    assertNotNull(FileFactory.getDataOutputStream(filePath, 
FileFactory.FileType.VIEWFS));
+    DataOutputStream outputStream = FileFactory.getDataOutputStream(filePath, 
FileFactory.FileType.VIEWFS);
+    assertNotNull(outputStream);
+    outputStream.close();
   }
 
   @Test public void getDataOutputStreamForLocalType() throws IOException {
-    assertNotNull(FileFactory.getDataOutputStream(filePath, 
FileFactory.FileType.LOCAL));
+    DataOutputStream outputStream = FileFactory.getDataOutputStream(filePath, 
FileFactory.FileType.LOCAL);
+    assertNotNull(outputStream);
+    outputStream.close();
   }
 
   @Test public void testGetCarbonFile() throws IOException {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/8a9dd8b2/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java
----------------------------------------------------------------------
diff --git 
a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java
 
b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java
index 4018123..42d4afa 100644
--- 
a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java
+++ 
b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java
@@ -58,7 +58,7 @@ public class HDFSCarbonFileTest {
     @BeforeClass
     static public void setUp() throws IOException {
         Configuration config = new Configuration();
-//adding local hadoop configuration
+        // adding local hadoop configuration
         config.addResource(new Path("core-site.xml"));
         config.addResource(new Path("hdfs-site.xml"));
         fileName = "Test.carbondata"; //this path is HDFS path
@@ -75,6 +75,7 @@ public class HDFSCarbonFileTest {
             BufferedWriter br = new BufferedWriter(new OutputStreamWriter(os, 
"UTF-8"));
             br.write("Hello World");
             br.close();
+            os.close();
             fs.close();
 
             fileStatus = new FileStatus(12L, true, 60, 120l, 180L, new 
Path(fileName));

http://git-wip-us.apache.org/repos/asf/carbondata/blob/8a9dd8b2/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFileTest.java
----------------------------------------------------------------------
diff --git 
a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFileTest.java
 
b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFileTest.java
index 96ef106..14f9fe2 100644
--- 
a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFileTest.java
+++ 
b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFileTest.java
@@ -29,6 +29,7 @@ import sun.nio.ch.FileChannelImpl;
 import java.io.*;
 import java.nio.channels.ReadableByteChannel;
 import java.util.Objects;
+import java.util.UUID;
 
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
@@ -44,8 +45,8 @@ public class LocalCarbonFileTest {
 
     @BeforeClass
     static public void setUp() {
-        file = new File("Test.carbondata");
-        dir = new File("Testdir.carbondata");
+        file = new File("TestLocalCarbonFile");
+        dir = new File("TestLocalCarbonDir");
         if (!file.exists())
             try {
                 file.createNewFile();
@@ -60,6 +61,7 @@ public class LocalCarbonFileTest {
             byte[] bytes = "core java api".getBytes();
 
             oFile.write(bytes);
+            oFile.close();
         } catch (FileNotFoundException e) {
             e.printStackTrace();
             localCarbonFile = new LocalCarbonFile(file);
@@ -121,8 +123,9 @@ public class LocalCarbonFileTest {
     @Test
     public void testRenameForce() {
         localCarbonFile = new LocalCarbonFile(file);
-        assertTrue(localCarbonFile.renameForce("Testdb.carbon"));
-        File file1 = new File("Testdb.carbon");
+        String destFile = "TestRename" + UUID.randomUUID().toString();
+        assertTrue(localCarbonFile.renameForce(destFile));
+        File file1 = new File(destFile);
         if (file1.exists()) {
             file1.delete();
         }
@@ -131,7 +134,12 @@ public class LocalCarbonFileTest {
     @Test
     public void testRenameTo() {
         localCarbonFile = new LocalCarbonFile(file);
-        assertTrue(!localCarbonFile.renameTo("Testdb.carbon"));
+        String destFile = "TestRename" + UUID.randomUUID().toString();
+        assertTrue(!localCarbonFile.renameTo(destFile));
+        File file1 = new File(destFile);
+        if (file1.exists()) {
+            file1.delete();
+        }
     }
 
     @Test
@@ -463,6 +471,6 @@ public class LocalCarbonFileTest {
 
         localCarbonFile = new LocalCarbonFile("demo.txt");
 
-        assertEquals(localCarbonFile.renameForce("Test.carbondata"), true);
+        assertEquals(localCarbonFile.renameForce("renameToFile"), true);
     }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/8a9dd8b2/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesInsertTest.scala
----------------------------------------------------------------------
diff --git 
a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesInsertTest.scala
 
b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesInsertTest.scala
index f8cfa6b..45edd3d 100644
--- 
a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesInsertTest.scala
+++ 
b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesInsertTest.scala
@@ -23,6 +23,7 @@ import org.apache.spark.sql.test.util.QueryTest
 import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
 
 import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datastore.impl.FileFactory
 import org.apache.carbondata.core.util.CarbonProperties
 
 class BooleanDataTypesInsertTest extends QueryTest with BeforeAndAfterEach 
with BeforeAndAfterAll {
@@ -555,7 +556,7 @@ class BooleanDataTypesInsertTest extends QueryTest with 
BeforeAndAfterEach with
 
     sql(
       s"""
-         | LOAD DATA LOCAL INPATH '${storeLocation}'
+         | LOAD DATA LOCAL INPATH 
'${FileFactory.getUpdatedFilePath(storeLocation)}'
          | INTO TABLE hive_table
            """.stripMargin)
 
@@ -923,7 +924,7 @@ class BooleanDataTypesInsertTest extends QueryTest with 
BeforeAndAfterEach with
 
     sql(
       s"""
-         | LOAD DATA LOCAL INPATH '${storeLocation}'
+         | LOAD DATA LOCAL INPATH 
'${FileFactory.getUpdatedFilePath(storeLocation)}'
          | INTO TABLE hive_table
            """.stripMargin)
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/8a9dd8b2/processing/src/test/java/org/apache/carbondata/lcm/locks/LocalFileLockTest.java
----------------------------------------------------------------------
diff --git 
a/processing/src/test/java/org/apache/carbondata/lcm/locks/LocalFileLockTest.java
 
b/processing/src/test/java/org/apache/carbondata/lcm/locks/LocalFileLockTest.java
index 812db5c..8d5f3d4 100644
--- 
a/processing/src/test/java/org/apache/carbondata/lcm/locks/LocalFileLockTest.java
+++ 
b/processing/src/test/java/org/apache/carbondata/lcm/locks/LocalFileLockTest.java
@@ -65,7 +65,7 @@ public class LocalFileLockTest {
 
     Assert.assertTrue(localLock1.unlock());
     Assert.assertTrue(localLock2.lock());
-
+    Assert.assertTrue(localLock2.unlock());
   }
 
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/8a9dd8b2/processing/src/test/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormatTest.java
----------------------------------------------------------------------
diff --git 
a/processing/src/test/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormatTest.java
 
b/processing/src/test/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormatTest.java
index 14c680e..9a9c662 100644
--- 
a/processing/src/test/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormatTest.java
+++ 
b/processing/src/test/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormatTest.java
@@ -147,6 +147,7 @@ public class CSVInputFormatTest extends TestCase {
     FileOutputFormat.setOutputPath(job, new Path(output.getCanonicalPath()));
 
     Assert.assertTrue(job.waitForCompletion(true));
+    deleteOutput(output);
   }
 
   private void prepareConf(Configuration conf) {

Reply via email to