[33/54] [abbrv] carbondata git commit: [CARBONDATA-1114][Tests] Fix bugs in tests in windows env

2018-03-08 Thread ravipesala 
[CARBONDATA-1114][Tests] Fix bugs in tests in windows env

Fix bugs in tests that will cause failure under windows env

This closes #1994


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/859d71c1
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/859d71c1
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/859d71c1

Branch: refs/heads/master
Commit: 859d71c1737b6287c4f1a06f7cd9055d32ff8a99
Parents: d5396b1
Author: xuchuanyin 
Authored: Sat Feb 24 21:18:17 2018 +0800
Committer: ravipesala 
Committed: Thu Mar 8 22:21:11 2018 +0530

--
 .../carbondata/core/datamap/dev/DataMap.java|   6 -
 .../core/datamap/dev/DataMapFactory.java|   2 +-
 .../exception/ConcurrentOperationException.java |  16 +-
 .../carbondata/core/locks/LocalFileLock.java|  30 ++--
 .../core/metadata/PartitionMapFileStore.java|   0
 .../statusmanager/SegmentStatusManager.java |  10 +-
 .../SegmentUpdateStatusManager.java |   1 -
 .../store/impl/DFSFileReaderImplUnitTest.java   |  11 +-
 .../store/impl/FileFactoryImplUnitTest.java |  28 +++-
 .../filesystem/HDFSCarbonFileTest.java  |   3 +-
 .../filesystem/LocalCarbonFileTest.java |  20 ++-
 datamap/examples/pom.xml| 145 +++--
 .../datamap/examples/MinMaxDataWriter.java  |   1 -
 examples/flink/pom.xml  |   4 +-
 .../carbondata/examples/FlinkExample.scala  |  10 +-
 .../CarbonStreamSparkStreamingExample.scala |   1 -
 .../hadoop/api/CarbonTableInputFormat.java  |   5 +-
 .../TestInsertAndOtherCommandConcurrent.scala   |   2 +-
 .../StandardPartitionGlobalSortTestCase.scala   |   2 +-
 .../exception/ProcessMetaDataException.java |   2 +
 .../org/apache/carbondata/api/CarbonStore.scala |   6 +-
 .../carbondata/spark/load/CsvRDDHelper.scala| 157 +++
 .../load/DataLoadProcessBuilderOnSpark.scala|   3 +-
 .../carbondata/spark/util/CarbonScalaUtil.scala |   2 +-
 .../carbondata/spark/util/CommonUtil.scala  |   2 -
 .../command/carbonTableSchemaCommon.scala   |   6 +-
 .../CarbonAlterTableCompactionCommand.scala |   3 +-
 .../management/CarbonCleanFilesCommand.scala|   2 +-
 .../CarbonDeleteLoadByIdCommand.scala   |   2 +-
 .../CarbonDeleteLoadByLoadDateCommand.scala |   2 +-
 .../management/CarbonLoadDataCommand.scala  |  28 ++--
 .../CarbonProjectForDeleteCommand.scala |   2 +-
 .../CarbonProjectForUpdateCommand.scala |   2 +-
 .../schema/CarbonAlterTableRenameCommand.scala  |   2 +-
 .../command/table/CarbonDropTableCommand.scala  |   2 +-
 .../datasources/CarbonFileFormat.scala  |   3 -
 .../BooleanDataTypesInsertTest.scala|   5 +-
 .../vectorreader/AddColumnTestCases.scala   |   1 +
 .../datamap/DataMapWriterListener.java  |   3 +-
 .../loading/model/CarbonLoadModelBuilder.java   |  34 +++-
 .../processing/loading/model/LoadOption.java|  15 +-
 .../processing/merger/CarbonDataMergerUtil.java |   3 +-
 .../util/CarbonDataProcessorUtil.java   |   3 +-
 .../processing/util/CarbonLoaderUtil.java   |   8 +
 .../carbondata/lcm/locks/LocalFileLockTest.java |   2 +-
 .../loading/csvinput/CSVInputFormatTest.java|   1 +
 store/sdk/pom.xml   |   2 +-
 .../carbondata/sdk/file/CSVCarbonWriter.java|   8 +-
 48 files changed, 400 insertions(+), 208 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java 
b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
index 02db8af..dd5507c 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
@@ -38,9 +38,6 @@ public interface DataMap {
   /**
* Prune the datamap with filter expression and partition information. It 
returns the list of
* blocklets where these filters can exist.
-   *
-   * @param filterExp
-   * @return
*/
   List prune(FilterResolverIntf filterExp, SegmentProperties 
segmentProperties,
   List partitions);
@@ -48,9 +45,6 @@ public interface DataMap {
   // TODO Move this method to Abstract class
   /**
* Validate whether the current segment needs to be fetching the required 
data
-   *
-   * @param filterExp
-   * @return
*/
   boolean isScanRequired(FilterResolverIntf filterExp);

[32/54] [abbrv] carbondata git commit: [CARBONDATA-1114][Tests] Fix bugs in tests in windows env

2018-03-08 Thread ravipesala 
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/processing/src/test/java/org/apache/carbondata/lcm/locks/LocalFileLockTest.java
--
diff --git 
a/processing/src/test/java/org/apache/carbondata/lcm/locks/LocalFileLockTest.java
 
b/processing/src/test/java/org/apache/carbondata/lcm/locks/LocalFileLockTest.java
index 812db5c..8d5f3d4 100644
--- 
a/processing/src/test/java/org/apache/carbondata/lcm/locks/LocalFileLockTest.java
+++ 
b/processing/src/test/java/org/apache/carbondata/lcm/locks/LocalFileLockTest.java
@@ -65,7 +65,7 @@ public class LocalFileLockTest {
 
 Assert.assertTrue(localLock1.unlock());
 Assert.assertTrue(localLock2.lock());
-
+Assert.assertTrue(localLock2.unlock());
   }
 
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/processing/src/test/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormatTest.java
--
diff --git 
a/processing/src/test/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormatTest.java
 
b/processing/src/test/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormatTest.java
index d89f10d..30d9da4 100644
--- 
a/processing/src/test/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormatTest.java
+++ 
b/processing/src/test/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormatTest.java
@@ -176,6 +176,7 @@ public class CSVInputFormatTest extends TestCase {
 FileOutputFormat.setOutputPath(job, new Path(output.getCanonicalPath()));
 
 Assert.assertTrue(job.waitForCompletion(true));
+deleteOutput(output);
   }
 
   private void prepareConf(Configuration conf) {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/store/sdk/pom.xml
--
diff --git a/store/sdk/pom.xml b/store/sdk/pom.xml
index 54fba55..b3dd464 100644
--- a/store/sdk/pom.xml
+++ b/store/sdk/pom.xml
@@ -7,7 +7,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.3.0-SNAPSHOT
+1.4.0-SNAPSHOT
 ../../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CSVCarbonWriter.java
--
diff --git 
a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CSVCarbonWriter.java 
b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CSVCarbonWriter.java
index dc5696a..df6afc6 100644
--- 
a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CSVCarbonWriter.java
+++ 
b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CSVCarbonWriter.java
@@ -23,7 +23,7 @@ import java.util.UUID;
 
 import org.apache.carbondata.common.annotations.InterfaceAudience;
 import org.apache.carbondata.hadoop.api.CarbonTableOutputFormat;
-import org.apache.carbondata.processing.loading.csvinput.StringArrayWritable;
+import org.apache.carbondata.hadoop.internal.ObjectArrayWritable;
 import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
 
 import org.apache.hadoop.conf.Configuration;
@@ -42,9 +42,9 @@ import 
org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
 @InterfaceAudience.Internal
 class CSVCarbonWriter extends CarbonWriter {
 
-  private RecordWriter recordWriter;
+  private RecordWriter recordWriter;
   private TaskAttemptContext context;
-  private StringArrayWritable writable;
+  private ObjectArrayWritable writable;
 
   CSVCarbonWriter(CarbonLoadModel loadModel) throws IOException {
 Configuration hadoopConf = new Configuration();
@@ -57,7 +57,7 @@ class CSVCarbonWriter extends CarbonWriter {
 TaskAttemptContextImpl context = new TaskAttemptContextImpl(hadoopConf, 
attemptID);
 this.recordWriter = format.getRecordWriter(context);
 this.context = context;
-this.writable = new StringArrayWritable();
+this.writable = new ObjectArrayWritable();
   }
 
   /**

[48/50] [abbrv] carbondata git commit: [CARBONDATA-1114][Tests] Fix bugs in tests in windows env

2018-03-04 Thread jackylk 
[CARBONDATA-1114][Tests] Fix bugs in tests in windows env

Fix bugs in tests that will cause failure under windows env

This closes #1994


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/ce88eb6a
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/ce88eb6a
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/ce88eb6a

Branch: refs/heads/carbonstore
Commit: ce88eb6a2d6d54acf15a2bdf2a9165ecc9570647
Parents: faad967
Author: xuchuanyin 
Authored: Sat Feb 24 21:18:17 2018 +0800
Committer: Jacky Li 
Committed: Sun Mar 4 20:32:14 2018 +0800

--
 .../carbondata/core/locks/LocalFileLock.java| 30 +++-
 .../store/impl/DFSFileReaderImplUnitTest.java   | 11 ---
 .../store/impl/FileFactoryImplUnitTest.java | 28 +-
 .../filesystem/HDFSCarbonFileTest.java  |  3 +-
 .../filesystem/LocalCarbonFileTest.java | 20 +
 .../BooleanDataTypesInsertTest.scala|  5 ++--
 .../carbondata/lcm/locks/LocalFileLockTest.java |  2 +-
 .../loading/csvinput/CSVInputFormatTest.java|  1 +
 8 files changed, 64 insertions(+), 36 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/ce88eb6a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java 
b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
index 75ea074..cb80877 100644
--- a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
+++ b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
@@ -17,17 +17,20 @@
 
 package org.apache.carbondata.core.locks;
 
-import java.io.FileOutputStream;
 import java.io.IOException;
 import java.nio.channels.FileChannel;
 import java.nio.channels.FileLock;
 import java.nio.channels.OverlappingFileLockException;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
 
 import org.apache.carbondata.common.logging.LogService;
 import org.apache.carbondata.common.logging.LogServiceFactory;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
 import org.apache.carbondata.core.datastore.impl.FileFactory;
 import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.util.CarbonUtil;
 
 /**
  * This class handles the file locking in the local file system.
@@ -40,11 +43,6 @@ public class LocalFileLock extends AbstractCarbonLock {
   private String location;
 
   /**
-   * fileOutputStream of the local lock file
-   */
-  private FileOutputStream fileOutputStream;
-
-  /**
* channel is the FileChannel of the lock file.
*/
   private FileChannel channel;
@@ -104,8 +102,8 @@ public class LocalFileLock extends AbstractCarbonLock {
 FileFactory.createNewLockFile(lockFilePath, 
FileFactory.getFileType(location));
   }
 
-  fileOutputStream = new FileOutputStream(lockFilePath);
-  channel = fileOutputStream.getChannel();
+  channel = FileChannel.open(Paths.get(lockFilePath), 
StandardOpenOption.WRITE,
+  StandardOpenOption.APPEND);
   try {
 fileLock = channel.tryLock();
   } catch (OverlappingFileLockException e) {
@@ -137,11 +135,17 @@ public class LocalFileLock extends AbstractCarbonLock {
 } catch (IOException e) {
   status = false;
 } finally {
-  if (null != fileOutputStream) {
-try {
-  fileOutputStream.close();
-} catch (IOException e) {
-  LOGGER.error(e.getMessage());
+  CarbonUtil.closeStreams(channel);
+
+  // deleting the lock file after releasing the lock.
+  if (null != lockFilePath) {
+CarbonFile lockFile = FileFactory.getCarbonFile(lockFilePath,
+FileFactory.getFileType(lockFilePath));
+if (!lockFile.exists() || lockFile.delete()) {
+  LOGGER.info("Successfully deleted the lock file " + lockFilePath);
+} else {
+  LOGGER.error("Not able to delete the lock file " + lockFilePath);
+  status = false;
 }
   }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/ce88eb6a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
--
diff --git 
a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
 
b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
index da61a94..30144c1

[49/50] [abbrv] carbondata git commit: [CARBONDATA-1114][Tests] Fix bugs in tests in windows env

2018-03-04 Thread jackylk 
[CARBONDATA-1114][Tests] Fix bugs in tests in windows env

Fix bugs in tests that will cause failure under windows env

This closes #1994


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/87bda960
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/87bda960
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/87bda960

Branch: refs/heads/carbonstore-rebase5
Commit: 87bda9601ed23871454640432025858799bff1f6
Parents: 49d06c2
Author: xuchuanyin 
Authored: Sat Feb 24 21:18:17 2018 +0800
Committer: Jacky Li 
Committed: Sun Mar 4 20:08:31 2018 +0800

--
 .../carbondata/core/locks/LocalFileLock.java| 30 +++-
 .../store/impl/DFSFileReaderImplUnitTest.java   | 11 ---
 .../store/impl/FileFactoryImplUnitTest.java | 28 +-
 .../filesystem/HDFSCarbonFileTest.java  |  3 +-
 .../filesystem/LocalCarbonFileTest.java | 20 +
 .../BooleanDataTypesInsertTest.scala|  5 ++--
 .../carbondata/lcm/locks/LocalFileLockTest.java |  2 +-
 .../loading/csvinput/CSVInputFormatTest.java|  1 +
 8 files changed, 64 insertions(+), 36 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/87bda960/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java 
b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
index 75ea074..cb80877 100644
--- a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
+++ b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
@@ -17,17 +17,20 @@
 
 package org.apache.carbondata.core.locks;
 
-import java.io.FileOutputStream;
 import java.io.IOException;
 import java.nio.channels.FileChannel;
 import java.nio.channels.FileLock;
 import java.nio.channels.OverlappingFileLockException;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
 
 import org.apache.carbondata.common.logging.LogService;
 import org.apache.carbondata.common.logging.LogServiceFactory;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
 import org.apache.carbondata.core.datastore.impl.FileFactory;
 import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.util.CarbonUtil;
 
 /**
  * This class handles the file locking in the local file system.
@@ -40,11 +43,6 @@ public class LocalFileLock extends AbstractCarbonLock {
   private String location;
 
   /**
-   * fileOutputStream of the local lock file
-   */
-  private FileOutputStream fileOutputStream;
-
-  /**
* channel is the FileChannel of the lock file.
*/
   private FileChannel channel;
@@ -104,8 +102,8 @@ public class LocalFileLock extends AbstractCarbonLock {
 FileFactory.createNewLockFile(lockFilePath, 
FileFactory.getFileType(location));
   }
 
-  fileOutputStream = new FileOutputStream(lockFilePath);
-  channel = fileOutputStream.getChannel();
+  channel = FileChannel.open(Paths.get(lockFilePath), 
StandardOpenOption.WRITE,
+  StandardOpenOption.APPEND);
   try {
 fileLock = channel.tryLock();
   } catch (OverlappingFileLockException e) {
@@ -137,11 +135,17 @@ public class LocalFileLock extends AbstractCarbonLock {
 } catch (IOException e) {
   status = false;
 } finally {
-  if (null != fileOutputStream) {
-try {
-  fileOutputStream.close();
-} catch (IOException e) {
-  LOGGER.error(e.getMessage());
+  CarbonUtil.closeStreams(channel);
+
+  // deleting the lock file after releasing the lock.
+  if (null != lockFilePath) {
+CarbonFile lockFile = FileFactory.getCarbonFile(lockFilePath,
+FileFactory.getFileType(lockFilePath));
+if (!lockFile.exists() || lockFile.delete()) {
+  LOGGER.info("Successfully deleted the lock file " + lockFilePath);
+} else {
+  LOGGER.error("Not able to delete the lock file " + lockFilePath);
+  status = false;
 }
   }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/87bda960/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
--
diff --git 
a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
 
b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
index

[48/50] [abbrv] carbondata git commit: [CARBONDATA-1114][Tests] Fix bugs in tests in windows env

2018-03-02 Thread jackylk 
[CARBONDATA-1114][Tests] Fix bugs in tests in windows env

Fix bugs in tests that will cause failure under windows env

This closes #1994


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/8a9dd8b2
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/8a9dd8b2
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/8a9dd8b2

Branch: refs/heads/carbonstore-rebase5
Commit: 8a9dd8b21907096ded81fb58e152eecc3f629bb8
Parents: b1c9628
Author: xuchuanyin 
Authored: Sat Feb 24 21:18:17 2018 +0800
Committer: Jacky Li 
Committed: Fri Mar 2 15:52:37 2018 +0800

--
 .../carbondata/core/locks/LocalFileLock.java| 30 +++-
 .../store/impl/DFSFileReaderImplUnitTest.java   | 11 ---
 .../store/impl/FileFactoryImplUnitTest.java | 28 +-
 .../filesystem/HDFSCarbonFileTest.java  |  3 +-
 .../filesystem/LocalCarbonFileTest.java | 20 +
 .../BooleanDataTypesInsertTest.scala|  5 ++--
 .../carbondata/lcm/locks/LocalFileLockTest.java |  2 +-
 .../loading/csvinput/CSVInputFormatTest.java|  1 +
 8 files changed, 64 insertions(+), 36 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/8a9dd8b2/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java 
b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
index 75ea074..cb80877 100644
--- a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
+++ b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
@@ -17,17 +17,20 @@
 
 package org.apache.carbondata.core.locks;
 
-import java.io.FileOutputStream;
 import java.io.IOException;
 import java.nio.channels.FileChannel;
 import java.nio.channels.FileLock;
 import java.nio.channels.OverlappingFileLockException;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
 
 import org.apache.carbondata.common.logging.LogService;
 import org.apache.carbondata.common.logging.LogServiceFactory;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
 import org.apache.carbondata.core.datastore.impl.FileFactory;
 import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.util.CarbonUtil;
 
 /**
  * This class handles the file locking in the local file system.
@@ -40,11 +43,6 @@ public class LocalFileLock extends AbstractCarbonLock {
   private String location;
 
   /**
-   * fileOutputStream of the local lock file
-   */
-  private FileOutputStream fileOutputStream;
-
-  /**
* channel is the FileChannel of the lock file.
*/
   private FileChannel channel;
@@ -104,8 +102,8 @@ public class LocalFileLock extends AbstractCarbonLock {
 FileFactory.createNewLockFile(lockFilePath, 
FileFactory.getFileType(location));
   }
 
-  fileOutputStream = new FileOutputStream(lockFilePath);
-  channel = fileOutputStream.getChannel();
+  channel = FileChannel.open(Paths.get(lockFilePath), 
StandardOpenOption.WRITE,
+  StandardOpenOption.APPEND);
   try {
 fileLock = channel.tryLock();
   } catch (OverlappingFileLockException e) {
@@ -137,11 +135,17 @@ public class LocalFileLock extends AbstractCarbonLock {
 } catch (IOException e) {
   status = false;
 } finally {
-  if (null != fileOutputStream) {
-try {
-  fileOutputStream.close();
-} catch (IOException e) {
-  LOGGER.error(e.getMessage());
+  CarbonUtil.closeStreams(channel);
+
+  // deleting the lock file after releasing the lock.
+  if (null != lockFilePath) {
+CarbonFile lockFile = FileFactory.getCarbonFile(lockFilePath,
+FileFactory.getFileType(lockFilePath));
+if (!lockFile.exists() || lockFile.delete()) {
+  LOGGER.info("Successfully deleted the lock file " + lockFilePath);
+} else {
+  LOGGER.error("Not able to delete the lock file " + lockFilePath);
+  status = false;
 }
   }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/8a9dd8b2/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
--
diff --git 
a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
 
b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
index

[46/49] carbondata git commit: [CARBONDATA-1114][Tests] Fix bugs in tests in windows env

2018-02-28 Thread jackylk 
[CARBONDATA-1114][Tests] Fix bugs in tests in windows env

Fix bugs in tests that will cause failure under windows env

This closes #1994


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/58c17539
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/58c17539
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/58c17539

Branch: refs/heads/carbonstore-rebase5
Commit: 58c17539b468e6ae089dfc51fbf3af85698047fc
Parents: 2414fb0
Author: xuchuanyin 
Authored: Sat Feb 24 21:18:17 2018 +0800
Committer: Jacky Li 
Committed: Wed Feb 28 22:05:23 2018 +0800

--
 .../carbondata/core/locks/LocalFileLock.java| 30 +++-
 .../store/impl/DFSFileReaderImplUnitTest.java   | 11 ---
 .../store/impl/FileFactoryImplUnitTest.java | 28 +-
 .../filesystem/HDFSCarbonFileTest.java  |  3 +-
 .../filesystem/LocalCarbonFileTest.java | 20 +
 .../BooleanDataTypesInsertTest.scala|  5 ++--
 .../carbondata/lcm/locks/LocalFileLockTest.java |  2 +-
 .../loading/csvinput/CSVInputFormatTest.java|  1 +
 8 files changed, 64 insertions(+), 36 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/58c17539/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java 
b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
index 75ea074..cb80877 100644
--- a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
+++ b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
@@ -17,17 +17,20 @@
 
 package org.apache.carbondata.core.locks;
 
-import java.io.FileOutputStream;
 import java.io.IOException;
 import java.nio.channels.FileChannel;
 import java.nio.channels.FileLock;
 import java.nio.channels.OverlappingFileLockException;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
 
 import org.apache.carbondata.common.logging.LogService;
 import org.apache.carbondata.common.logging.LogServiceFactory;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
 import org.apache.carbondata.core.datastore.impl.FileFactory;
 import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.util.CarbonUtil;
 
 /**
  * This class handles the file locking in the local file system.
@@ -40,11 +43,6 @@ public class LocalFileLock extends AbstractCarbonLock {
   private String location;
 
   /**
-   * fileOutputStream of the local lock file
-   */
-  private FileOutputStream fileOutputStream;
-
-  /**
* channel is the FileChannel of the lock file.
*/
   private FileChannel channel;
@@ -104,8 +102,8 @@ public class LocalFileLock extends AbstractCarbonLock {
 FileFactory.createNewLockFile(lockFilePath, 
FileFactory.getFileType(location));
   }
 
-  fileOutputStream = new FileOutputStream(lockFilePath);
-  channel = fileOutputStream.getChannel();
+  channel = FileChannel.open(Paths.get(lockFilePath), 
StandardOpenOption.WRITE,
+  StandardOpenOption.APPEND);
   try {
 fileLock = channel.tryLock();
   } catch (OverlappingFileLockException e) {
@@ -137,11 +135,17 @@ public class LocalFileLock extends AbstractCarbonLock {
 } catch (IOException e) {
   status = false;
 } finally {
-  if (null != fileOutputStream) {
-try {
-  fileOutputStream.close();
-} catch (IOException e) {
-  LOGGER.error(e.getMessage());
+  CarbonUtil.closeStreams(channel);
+
+  // deleting the lock file after releasing the lock.
+  if (null != lockFilePath) {
+CarbonFile lockFile = FileFactory.getCarbonFile(lockFilePath,
+FileFactory.getFileType(lockFilePath));
+if (!lockFile.exists() || lockFile.delete()) {
+  LOGGER.info("Successfully deleted the lock file " + lockFilePath);
+} else {
+  LOGGER.error("Not able to delete the lock file " + lockFilePath);
+  status = false;
 }
   }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/58c17539/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
--
diff --git 
a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
 
b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
index

[49/50] carbondata git commit: [CARBONDATA-1114][Tests] Fix bugs in tests in windows env

2018-02-27 Thread jackylk 
[CARBONDATA-1114][Tests] Fix bugs in tests in windows env

Fix bugs in tests that will cause failure under windows env

This closes #1994


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/c45a44d9
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/c45a44d9
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/c45a44d9

Branch: refs/heads/carbonstore-rebase4
Commit: c45a44d932fd565a39f9b128c0269c4e98d09b99
Parents: e5e187f
Author: xuchuanyin 
Authored: Sat Feb 24 21:18:17 2018 +0800
Committer: Jacky Li 
Committed: Tue Feb 27 17:04:20 2018 +0800

--
 .../carbondata/core/locks/LocalFileLock.java| 30 +++-
 .../store/impl/DFSFileReaderImplUnitTest.java   | 11 ---
 .../store/impl/FileFactoryImplUnitTest.java | 28 +-
 .../filesystem/HDFSCarbonFileTest.java  |  3 +-
 .../filesystem/LocalCarbonFileTest.java | 20 +
 .../BooleanDataTypesInsertTest.scala|  5 ++--
 .../carbondata/lcm/locks/LocalFileLockTest.java |  2 +-
 .../loading/csvinput/CSVInputFormatTest.java|  1 +
 8 files changed, 64 insertions(+), 36 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/c45a44d9/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java 
b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
index 75ea074..cb80877 100644
--- a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
+++ b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
@@ -17,17 +17,20 @@
 
 package org.apache.carbondata.core.locks;
 
-import java.io.FileOutputStream;
 import java.io.IOException;
 import java.nio.channels.FileChannel;
 import java.nio.channels.FileLock;
 import java.nio.channels.OverlappingFileLockException;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
 
 import org.apache.carbondata.common.logging.LogService;
 import org.apache.carbondata.common.logging.LogServiceFactory;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
 import org.apache.carbondata.core.datastore.impl.FileFactory;
 import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.util.CarbonUtil;
 
 /**
  * This class handles the file locking in the local file system.
@@ -40,11 +43,6 @@ public class LocalFileLock extends AbstractCarbonLock {
   private String location;
 
   /**
-   * fileOutputStream of the local lock file
-   */
-  private FileOutputStream fileOutputStream;
-
-  /**
* channel is the FileChannel of the lock file.
*/
   private FileChannel channel;
@@ -104,8 +102,8 @@ public class LocalFileLock extends AbstractCarbonLock {
 FileFactory.createNewLockFile(lockFilePath, 
FileFactory.getFileType(location));
   }
 
-  fileOutputStream = new FileOutputStream(lockFilePath);
-  channel = fileOutputStream.getChannel();
+  channel = FileChannel.open(Paths.get(lockFilePath), 
StandardOpenOption.WRITE,
+  StandardOpenOption.APPEND);
   try {
 fileLock = channel.tryLock();
   } catch (OverlappingFileLockException e) {
@@ -137,11 +135,17 @@ public class LocalFileLock extends AbstractCarbonLock {
 } catch (IOException e) {
   status = false;
 } finally {
-  if (null != fileOutputStream) {
-try {
-  fileOutputStream.close();
-} catch (IOException e) {
-  LOGGER.error(e.getMessage());
+  CarbonUtil.closeStreams(channel);
+
+  // deleting the lock file after releasing the lock.
+  if (null != lockFilePath) {
+CarbonFile lockFile = FileFactory.getCarbonFile(lockFilePath,
+FileFactory.getFileType(lockFilePath));
+if (!lockFile.exists() || lockFile.delete()) {
+  LOGGER.info("Successfully deleted the lock file " + lockFilePath);
+} else {
+  LOGGER.error("Not able to delete the lock file " + lockFilePath);
+  status = false;
 }
   }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/c45a44d9/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
--
diff --git 
a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
 
b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
index

[48/49] carbondata git commit: [CARBONDATA-1114][Tests] Fix bugs in tests in windows env

2018-02-26 Thread jackylk 
[CARBONDATA-1114][Tests] Fix bugs in tests in windows env

Fix bugs in tests that will cause failure under windows env

This closes #1994


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/de36a5d8
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/de36a5d8
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/de36a5d8

Branch: refs/heads/carbonstore-rebase4
Commit: de36a5d85d992dee4380ae228e6252baf1c31fe9
Parents: fa1a100
Author: xuchuanyin 
Authored: Sat Feb 24 21:18:17 2018 +0800
Committer: Jacky Li 
Committed: Tue Feb 27 09:24:02 2018 +0800

--
 .../carbondata/core/locks/LocalFileLock.java| 30 +++-
 .../store/impl/DFSFileReaderImplUnitTest.java   | 11 ---
 .../store/impl/FileFactoryImplUnitTest.java | 28 +-
 .../filesystem/HDFSCarbonFileTest.java  |  3 +-
 .../filesystem/LocalCarbonFileTest.java | 20 +
 .../BooleanDataTypesInsertTest.scala|  5 ++--
 .../carbondata/lcm/locks/LocalFileLockTest.java |  2 +-
 .../loading/csvinput/CSVInputFormatTest.java|  1 +
 8 files changed, 64 insertions(+), 36 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/de36a5d8/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java 
b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
index 75ea074..cb80877 100644
--- a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
+++ b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
@@ -17,17 +17,20 @@
 
 package org.apache.carbondata.core.locks;
 
-import java.io.FileOutputStream;
 import java.io.IOException;
 import java.nio.channels.FileChannel;
 import java.nio.channels.FileLock;
 import java.nio.channels.OverlappingFileLockException;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
 
 import org.apache.carbondata.common.logging.LogService;
 import org.apache.carbondata.common.logging.LogServiceFactory;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
 import org.apache.carbondata.core.datastore.impl.FileFactory;
 import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.util.CarbonUtil;
 
 /**
  * This class handles the file locking in the local file system.
@@ -40,11 +43,6 @@ public class LocalFileLock extends AbstractCarbonLock {
   private String location;
 
   /**
-   * fileOutputStream of the local lock file
-   */
-  private FileOutputStream fileOutputStream;
-
-  /**
* channel is the FileChannel of the lock file.
*/
   private FileChannel channel;
@@ -104,8 +102,8 @@ public class LocalFileLock extends AbstractCarbonLock {
 FileFactory.createNewLockFile(lockFilePath, 
FileFactory.getFileType(location));
   }
 
-  fileOutputStream = new FileOutputStream(lockFilePath);
-  channel = fileOutputStream.getChannel();
+  channel = FileChannel.open(Paths.get(lockFilePath), 
StandardOpenOption.WRITE,
+  StandardOpenOption.APPEND);
   try {
 fileLock = channel.tryLock();
   } catch (OverlappingFileLockException e) {
@@ -137,11 +135,17 @@ public class LocalFileLock extends AbstractCarbonLock {
 } catch (IOException e) {
   status = false;
 } finally {
-  if (null != fileOutputStream) {
-try {
-  fileOutputStream.close();
-} catch (IOException e) {
-  LOGGER.error(e.getMessage());
+  CarbonUtil.closeStreams(channel);
+
+  // deleting the lock file after releasing the lock.
+  if (null != lockFilePath) {
+CarbonFile lockFile = FileFactory.getCarbonFile(lockFilePath,
+FileFactory.getFileType(lockFilePath));
+if (!lockFile.exists() || lockFile.delete()) {
+  LOGGER.info("Successfully deleted the lock file " + lockFilePath);
+} else {
+  LOGGER.error("Not able to delete the lock file " + lockFilePath);
+  status = false;
 }
   }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/de36a5d8/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
--
diff --git 
a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
 
b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
index

carbondata git commit: [CARBONDATA-1114][Tests] Fix bugs in tests in windows env

2018-02-26 Thread jackylk 
Repository: carbondata
Updated Branches:
  refs/heads/carbonstore ab9b4cf89 -> 37080e9a3


[CARBONDATA-1114][Tests] Fix bugs in tests in windows env

Fix bugs in tests that will cause failure under windows env

This closes #1994


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/37080e9a
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/37080e9a
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/37080e9a

Branch: refs/heads/carbonstore
Commit: 37080e9a3eb9472265eea1b0387c3e4f3047619a
Parents: ab9b4cf
Author: xuchuanyin 
Authored: Sat Feb 24 21:18:17 2018 +0800
Committer: Jacky Li 
Committed: Mon Feb 26 23:30:00 2018 +0800

--
 .../carbondata/core/locks/LocalFileLock.java| 38 +---
 .../store/impl/DFSFileReaderImplUnitTest.java   | 11 +++---
 .../store/impl/FileFactoryImplUnitTest.java | 28 +++
 .../filesystem/HDFSCarbonFileTest.java  |  3 +-
 .../filesystem/LocalCarbonFileTest.java | 20 +++
 .../BooleanDataTypesInsertTest.scala|  5 +--
 .../carbondata/lcm/locks/LocalFileLockTest.java |  2 +-
 .../loading/csvinput/CSVInputFormatTest.java|  1 +
 8 files changed, 63 insertions(+), 45 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/37080e9a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java 
b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
index c3dfb57..793d074 100644
--- a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
+++ b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
@@ -17,11 +17,12 @@
 
 package org.apache.carbondata.core.locks;
 
-import java.io.FileOutputStream;
 import java.io.IOException;
 import java.nio.channels.FileChannel;
 import java.nio.channels.FileLock;
 import java.nio.channels.OverlappingFileLockException;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
 
 import org.apache.carbondata.common.logging.LogService;
 import org.apache.carbondata.common.logging.LogServiceFactory;
@@ -29,6 +30,7 @@ import 
org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
 import org.apache.carbondata.core.datastore.impl.FileFactory;
 import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.util.CarbonUtil;
 
 /**
  * This class handles the file locking in the local file system.
@@ -41,11 +43,6 @@ public class LocalFileLock extends AbstractCarbonLock {
   private String location;
 
   /**
-   * fileOutputStream of the local lock file
-   */
-  private FileOutputStream fileOutputStream;
-
-  /**
* channel is the FileChannel of the lock file.
*/
   private FileChannel channel;
@@ -105,8 +102,8 @@ public class LocalFileLock extends AbstractCarbonLock {
 FileFactory.createNewLockFile(lockFilePath, 
FileFactory.getFileType(location));
   }
 
-  fileOutputStream = new FileOutputStream(lockFilePath);
-  channel = fileOutputStream.getChannel();
+  channel = FileChannel.open(Paths.get(lockFilePath), 
StandardOpenOption.WRITE,
+  StandardOpenOption.APPEND);
   try {
 fileLock = channel.tryLock();
   } catch (OverlappingFileLockException e) {
@@ -139,20 +136,17 @@ public class LocalFileLock extends AbstractCarbonLock {
 } catch (IOException e) {
   status = false;
 } finally {
-  if (null != fileOutputStream) {
-try {
-  fileOutputStream.close();
-  // deleting the lock file after releasing the lock.
-  CarbonFile lockFile = FileFactory
-  .getCarbonFile(lockFilePath, 
FileFactory.getFileType(lockFilePath));
-  if (!lockFile.exists() || lockFile.delete()) {
-LOGGER.info("Successfully deleted the lock file " + lockFilePath);
-  } else {
-LOGGER.error("Not able to delete the lock file " + lockFilePath);
-status = false;
-  }
-} catch (IOException e) {
-  LOGGER.error(e.getMessage());
+  CarbonUtil.closeStreams(channel);
+
+  // deleting the lock file after releasing the lock.
+  if (null != lockFilePath) {
+CarbonFile lockFile = FileFactory.getCarbonFile(lockFilePath,
+FileFactory.getFileType(lockFilePath));
+if (!lockFile.exists() || lockFile.delete()) {
+  LOGGER.info("Successfully deleted the lock file " + lockFilePath);
+} else {
+  LOGGER.error("Not able to delete the lock file " + lockFilePath);
+