[33/54] [abbrv] carbondata git commit: [CARBONDATA-1114][Tests] Fix bugs in tests in windows env
[CARBONDATA-1114][Tests] Fix bugs in tests in windows env Fix bugs in tests that will cause failure under windows env This closes #1994 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/859d71c1 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/859d71c1 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/859d71c1 Branch: refs/heads/master Commit: 859d71c1737b6287c4f1a06f7cd9055d32ff8a99 Parents: d5396b1 Author: xuchuanyin Authored: Sat Feb 24 21:18:17 2018 +0800 Committer: ravipesala Committed: Thu Mar 8 22:21:11 2018 +0530 -- .../carbondata/core/datamap/dev/DataMap.java| 6 - .../core/datamap/dev/DataMapFactory.java| 2 +- .../exception/ConcurrentOperationException.java | 16 +- .../carbondata/core/locks/LocalFileLock.java| 30 ++-- .../core/metadata/PartitionMapFileStore.java| 0 .../statusmanager/SegmentStatusManager.java | 10 +- .../SegmentUpdateStatusManager.java | 1 - .../store/impl/DFSFileReaderImplUnitTest.java | 11 +- .../store/impl/FileFactoryImplUnitTest.java | 28 +++- .../filesystem/HDFSCarbonFileTest.java | 3 +- .../filesystem/LocalCarbonFileTest.java | 20 ++- datamap/examples/pom.xml| 145 +++-- .../datamap/examples/MinMaxDataWriter.java | 1 - examples/flink/pom.xml | 4 +- .../carbondata/examples/FlinkExample.scala | 10 +- .../CarbonStreamSparkStreamingExample.scala | 1 - .../hadoop/api/CarbonTableInputFormat.java | 5 +- .../TestInsertAndOtherCommandConcurrent.scala | 2 +- .../StandardPartitionGlobalSortTestCase.scala | 2 +- .../exception/ProcessMetaDataException.java | 2 + .../org/apache/carbondata/api/CarbonStore.scala | 6 +- .../carbondata/spark/load/CsvRDDHelper.scala| 157 +++ .../load/DataLoadProcessBuilderOnSpark.scala| 3 +- .../carbondata/spark/util/CarbonScalaUtil.scala | 2 +- .../carbondata/spark/util/CommonUtil.scala | 2 - .../command/carbonTableSchemaCommon.scala | 6 +- .../CarbonAlterTableCompactionCommand.scala | 3 +- .../management/CarbonCleanFilesCommand.scala| 2 +- .../CarbonDeleteLoadByIdCommand.scala | 2 +- .../CarbonDeleteLoadByLoadDateCommand.scala | 2 +- .../management/CarbonLoadDataCommand.scala | 28 ++-- .../CarbonProjectForDeleteCommand.scala | 2 +- .../CarbonProjectForUpdateCommand.scala | 2 +- .../schema/CarbonAlterTableRenameCommand.scala | 2 +- .../command/table/CarbonDropTableCommand.scala | 2 +- .../datasources/CarbonFileFormat.scala | 3 - .../BooleanDataTypesInsertTest.scala| 5 +- .../vectorreader/AddColumnTestCases.scala | 1 + .../datamap/DataMapWriterListener.java | 3 +- .../loading/model/CarbonLoadModelBuilder.java | 34 +++- .../processing/loading/model/LoadOption.java| 15 +- .../processing/merger/CarbonDataMergerUtil.java | 3 +- .../util/CarbonDataProcessorUtil.java | 3 +- .../processing/util/CarbonLoaderUtil.java | 8 + .../carbondata/lcm/locks/LocalFileLockTest.java | 2 +- .../loading/csvinput/CSVInputFormatTest.java| 1 + store/sdk/pom.xml | 2 +- .../carbondata/sdk/file/CSVCarbonWriter.java| 8 +- 48 files changed, 400 insertions(+), 208 deletions(-) -- http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java -- diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java index 02db8af..dd5507c 100644 --- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java +++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java @@ -38,9 +38,6 @@ public interface DataMap { /** * Prune the datamap with filter expression and partition information. It returns the list of * blocklets where these filters can exist. - * - * @param filterExp - * @return */ List prune(FilterResolverIntf filterExp, SegmentProperties segmentProperties, List partitions); @@ -48,9 +45,6 @@ public interface DataMap { // TODO Move this method to Abstract class /** * Validate whether the current segment needs to be fetching the required data - * - * @param filterExp - * @return */ boolean isScanRequired(FilterResolverIntf filterExp); http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java ---
[32/54] [abbrv] carbondata git commit: [CARBONDATA-1114][Tests] Fix bugs in tests in windows env
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/processing/src/test/java/org/apache/carbondata/lcm/locks/LocalFileLockTest.java -- diff --git a/processing/src/test/java/org/apache/carbondata/lcm/locks/LocalFileLockTest.java b/processing/src/test/java/org/apache/carbondata/lcm/locks/LocalFileLockTest.java index 812db5c..8d5f3d4 100644 --- a/processing/src/test/java/org/apache/carbondata/lcm/locks/LocalFileLockTest.java +++ b/processing/src/test/java/org/apache/carbondata/lcm/locks/LocalFileLockTest.java @@ -65,7 +65,7 @@ public class LocalFileLockTest { Assert.assertTrue(localLock1.unlock()); Assert.assertTrue(localLock2.lock()); - +Assert.assertTrue(localLock2.unlock()); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/processing/src/test/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormatTest.java -- diff --git a/processing/src/test/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormatTest.java b/processing/src/test/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormatTest.java index d89f10d..30d9da4 100644 --- a/processing/src/test/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormatTest.java +++ b/processing/src/test/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormatTest.java @@ -176,6 +176,7 @@ public class CSVInputFormatTest extends TestCase { FileOutputFormat.setOutputPath(job, new Path(output.getCanonicalPath())); Assert.assertTrue(job.waitForCompletion(true)); +deleteOutput(output); } private void prepareConf(Configuration conf) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/store/sdk/pom.xml -- diff --git a/store/sdk/pom.xml b/store/sdk/pom.xml index 54fba55..b3dd464 100644 --- a/store/sdk/pom.xml +++ b/store/sdk/pom.xml @@ -7,7 +7,7 @@ org.apache.carbondata carbondata-parent -1.3.0-SNAPSHOT +1.4.0-SNAPSHOT ../../pom.xml http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CSVCarbonWriter.java -- diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CSVCarbonWriter.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CSVCarbonWriter.java index dc5696a..df6afc6 100644 --- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CSVCarbonWriter.java +++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CSVCarbonWriter.java @@ -23,7 +23,7 @@ import java.util.UUID; import org.apache.carbondata.common.annotations.InterfaceAudience; import org.apache.carbondata.hadoop.api.CarbonTableOutputFormat; -import org.apache.carbondata.processing.loading.csvinput.StringArrayWritable; +import org.apache.carbondata.hadoop.internal.ObjectArrayWritable; import org.apache.carbondata.processing.loading.model.CarbonLoadModel; import org.apache.hadoop.conf.Configuration; @@ -42,9 +42,9 @@ import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl; @InterfaceAudience.Internal class CSVCarbonWriter extends CarbonWriter { - private RecordWriter recordWriter; + private RecordWriter recordWriter; private TaskAttemptContext context; - private StringArrayWritable writable; + private ObjectArrayWritable writable; CSVCarbonWriter(CarbonLoadModel loadModel) throws IOException { Configuration hadoopConf = new Configuration(); @@ -57,7 +57,7 @@ class CSVCarbonWriter extends CarbonWriter { TaskAttemptContextImpl context = new TaskAttemptContextImpl(hadoopConf, attemptID); this.recordWriter = format.getRecordWriter(context); this.context = context; -this.writable = new StringArrayWritable(); +this.writable = new ObjectArrayWritable(); } /**
[48/50] [abbrv] carbondata git commit: [CARBONDATA-1114][Tests] Fix bugs in tests in windows env
[CARBONDATA-1114][Tests] Fix bugs in tests in windows env Fix bugs in tests that will cause failure under windows env This closes #1994 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/ce88eb6a Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/ce88eb6a Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/ce88eb6a Branch: refs/heads/carbonstore Commit: ce88eb6a2d6d54acf15a2bdf2a9165ecc9570647 Parents: faad967 Author: xuchuanyin Authored: Sat Feb 24 21:18:17 2018 +0800 Committer: Jacky Li Committed: Sun Mar 4 20:32:14 2018 +0800 -- .../carbondata/core/locks/LocalFileLock.java| 30 +++- .../store/impl/DFSFileReaderImplUnitTest.java | 11 --- .../store/impl/FileFactoryImplUnitTest.java | 28 +- .../filesystem/HDFSCarbonFileTest.java | 3 +- .../filesystem/LocalCarbonFileTest.java | 20 + .../BooleanDataTypesInsertTest.scala| 5 ++-- .../carbondata/lcm/locks/LocalFileLockTest.java | 2 +- .../loading/csvinput/CSVInputFormatTest.java| 1 + 8 files changed, 64 insertions(+), 36 deletions(-) -- http://git-wip-us.apache.org/repos/asf/carbondata/blob/ce88eb6a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java -- diff --git a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java index 75ea074..cb80877 100644 --- a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java +++ b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java @@ -17,17 +17,20 @@ package org.apache.carbondata.core.locks; -import java.io.FileOutputStream; import java.io.IOException; import java.nio.channels.FileChannel; import java.nio.channels.FileLock; import java.nio.channels.OverlappingFileLockException; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.datastore.filesystem.CarbonFile; import org.apache.carbondata.core.datastore.impl.FileFactory; import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; +import org.apache.carbondata.core.util.CarbonUtil; /** * This class handles the file locking in the local file system. @@ -40,11 +43,6 @@ public class LocalFileLock extends AbstractCarbonLock { private String location; /** - * fileOutputStream of the local lock file - */ - private FileOutputStream fileOutputStream; - - /** * channel is the FileChannel of the lock file. */ private FileChannel channel; @@ -104,8 +102,8 @@ public class LocalFileLock extends AbstractCarbonLock { FileFactory.createNewLockFile(lockFilePath, FileFactory.getFileType(location)); } - fileOutputStream = new FileOutputStream(lockFilePath); - channel = fileOutputStream.getChannel(); + channel = FileChannel.open(Paths.get(lockFilePath), StandardOpenOption.WRITE, + StandardOpenOption.APPEND); try { fileLock = channel.tryLock(); } catch (OverlappingFileLockException e) { @@ -137,11 +135,17 @@ public class LocalFileLock extends AbstractCarbonLock { } catch (IOException e) { status = false; } finally { - if (null != fileOutputStream) { -try { - fileOutputStream.close(); -} catch (IOException e) { - LOGGER.error(e.getMessage()); + CarbonUtil.closeStreams(channel); + + // deleting the lock file after releasing the lock. + if (null != lockFilePath) { +CarbonFile lockFile = FileFactory.getCarbonFile(lockFilePath, +FileFactory.getFileType(lockFilePath)); +if (!lockFile.exists() || lockFile.delete()) { + LOGGER.info("Successfully deleted the lock file " + lockFilePath); +} else { + LOGGER.error("Not able to delete the lock file " + lockFilePath); + status = false; } } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/ce88eb6a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java -- diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java index da61a94..30144c1 100644 --- a/core/src/test/java/org/apache/
[48/50] [abbrv] carbondata git commit: [CARBONDATA-1114][Tests] Fix bugs in tests in windows env
[CARBONDATA-1114][Tests] Fix bugs in tests in windows env Fix bugs in tests that will cause failure under windows env This closes #1994 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/ce88eb6a Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/ce88eb6a Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/ce88eb6a Branch: refs/heads/carbonstore-rebase5 Commit: ce88eb6a2d6d54acf15a2bdf2a9165ecc9570647 Parents: faad967 Author: xuchuanyin Authored: Sat Feb 24 21:18:17 2018 +0800 Committer: Jacky Li Committed: Sun Mar 4 20:32:14 2018 +0800 -- .../carbondata/core/locks/LocalFileLock.java| 30 +++- .../store/impl/DFSFileReaderImplUnitTest.java | 11 --- .../store/impl/FileFactoryImplUnitTest.java | 28 +- .../filesystem/HDFSCarbonFileTest.java | 3 +- .../filesystem/LocalCarbonFileTest.java | 20 + .../BooleanDataTypesInsertTest.scala| 5 ++-- .../carbondata/lcm/locks/LocalFileLockTest.java | 2 +- .../loading/csvinput/CSVInputFormatTest.java| 1 + 8 files changed, 64 insertions(+), 36 deletions(-) -- http://git-wip-us.apache.org/repos/asf/carbondata/blob/ce88eb6a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java -- diff --git a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java index 75ea074..cb80877 100644 --- a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java +++ b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java @@ -17,17 +17,20 @@ package org.apache.carbondata.core.locks; -import java.io.FileOutputStream; import java.io.IOException; import java.nio.channels.FileChannel; import java.nio.channels.FileLock; import java.nio.channels.OverlappingFileLockException; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.datastore.filesystem.CarbonFile; import org.apache.carbondata.core.datastore.impl.FileFactory; import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; +import org.apache.carbondata.core.util.CarbonUtil; /** * This class handles the file locking in the local file system. @@ -40,11 +43,6 @@ public class LocalFileLock extends AbstractCarbonLock { private String location; /** - * fileOutputStream of the local lock file - */ - private FileOutputStream fileOutputStream; - - /** * channel is the FileChannel of the lock file. */ private FileChannel channel; @@ -104,8 +102,8 @@ public class LocalFileLock extends AbstractCarbonLock { FileFactory.createNewLockFile(lockFilePath, FileFactory.getFileType(location)); } - fileOutputStream = new FileOutputStream(lockFilePath); - channel = fileOutputStream.getChannel(); + channel = FileChannel.open(Paths.get(lockFilePath), StandardOpenOption.WRITE, + StandardOpenOption.APPEND); try { fileLock = channel.tryLock(); } catch (OverlappingFileLockException e) { @@ -137,11 +135,17 @@ public class LocalFileLock extends AbstractCarbonLock { } catch (IOException e) { status = false; } finally { - if (null != fileOutputStream) { -try { - fileOutputStream.close(); -} catch (IOException e) { - LOGGER.error(e.getMessage()); + CarbonUtil.closeStreams(channel); + + // deleting the lock file after releasing the lock. + if (null != lockFilePath) { +CarbonFile lockFile = FileFactory.getCarbonFile(lockFilePath, +FileFactory.getFileType(lockFilePath)); +if (!lockFile.exists() || lockFile.delete()) { + LOGGER.info("Successfully deleted the lock file " + lockFilePath); +} else { + LOGGER.error("Not able to delete the lock file " + lockFilePath); + status = false; } } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/ce88eb6a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java -- diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java index da61a94..30144c1 100644 --- a/core/src/test/java/org
[49/50] [abbrv] carbondata git commit: [CARBONDATA-1114][Tests] Fix bugs in tests in windows env
[CARBONDATA-1114][Tests] Fix bugs in tests in windows env Fix bugs in tests that will cause failure under windows env This closes #1994 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/87bda960 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/87bda960 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/87bda960 Branch: refs/heads/carbonstore-rebase5 Commit: 87bda9601ed23871454640432025858799bff1f6 Parents: 49d06c2 Author: xuchuanyin Authored: Sat Feb 24 21:18:17 2018 +0800 Committer: Jacky Li Committed: Sun Mar 4 20:08:31 2018 +0800 -- .../carbondata/core/locks/LocalFileLock.java| 30 +++- .../store/impl/DFSFileReaderImplUnitTest.java | 11 --- .../store/impl/FileFactoryImplUnitTest.java | 28 +- .../filesystem/HDFSCarbonFileTest.java | 3 +- .../filesystem/LocalCarbonFileTest.java | 20 + .../BooleanDataTypesInsertTest.scala| 5 ++-- .../carbondata/lcm/locks/LocalFileLockTest.java | 2 +- .../loading/csvinput/CSVInputFormatTest.java| 1 + 8 files changed, 64 insertions(+), 36 deletions(-) -- http://git-wip-us.apache.org/repos/asf/carbondata/blob/87bda960/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java -- diff --git a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java index 75ea074..cb80877 100644 --- a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java +++ b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java @@ -17,17 +17,20 @@ package org.apache.carbondata.core.locks; -import java.io.FileOutputStream; import java.io.IOException; import java.nio.channels.FileChannel; import java.nio.channels.FileLock; import java.nio.channels.OverlappingFileLockException; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.datastore.filesystem.CarbonFile; import org.apache.carbondata.core.datastore.impl.FileFactory; import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; +import org.apache.carbondata.core.util.CarbonUtil; /** * This class handles the file locking in the local file system. @@ -40,11 +43,6 @@ public class LocalFileLock extends AbstractCarbonLock { private String location; /** - * fileOutputStream of the local lock file - */ - private FileOutputStream fileOutputStream; - - /** * channel is the FileChannel of the lock file. */ private FileChannel channel; @@ -104,8 +102,8 @@ public class LocalFileLock extends AbstractCarbonLock { FileFactory.createNewLockFile(lockFilePath, FileFactory.getFileType(location)); } - fileOutputStream = new FileOutputStream(lockFilePath); - channel = fileOutputStream.getChannel(); + channel = FileChannel.open(Paths.get(lockFilePath), StandardOpenOption.WRITE, + StandardOpenOption.APPEND); try { fileLock = channel.tryLock(); } catch (OverlappingFileLockException e) { @@ -137,11 +135,17 @@ public class LocalFileLock extends AbstractCarbonLock { } catch (IOException e) { status = false; } finally { - if (null != fileOutputStream) { -try { - fileOutputStream.close(); -} catch (IOException e) { - LOGGER.error(e.getMessage()); + CarbonUtil.closeStreams(channel); + + // deleting the lock file after releasing the lock. + if (null != lockFilePath) { +CarbonFile lockFile = FileFactory.getCarbonFile(lockFilePath, +FileFactory.getFileType(lockFilePath)); +if (!lockFile.exists() || lockFile.delete()) { + LOGGER.info("Successfully deleted the lock file " + lockFilePath); +} else { + LOGGER.error("Not able to delete the lock file " + lockFilePath); + status = false; } } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/87bda960/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java -- diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java index da61a94..30144c1 100644 --- a/core/src/test/java/org
[48/50] [abbrv] carbondata git commit: [CARBONDATA-1114][Tests] Fix bugs in tests in windows env
[CARBONDATA-1114][Tests] Fix bugs in tests in windows env Fix bugs in tests that will cause failure under windows env This closes #1994 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/8a9dd8b2 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/8a9dd8b2 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/8a9dd8b2 Branch: refs/heads/carbonstore-rebase5 Commit: 8a9dd8b21907096ded81fb58e152eecc3f629bb8 Parents: b1c9628 Author: xuchuanyin Authored: Sat Feb 24 21:18:17 2018 +0800 Committer: Jacky Li Committed: Fri Mar 2 15:52:37 2018 +0800 -- .../carbondata/core/locks/LocalFileLock.java| 30 +++- .../store/impl/DFSFileReaderImplUnitTest.java | 11 --- .../store/impl/FileFactoryImplUnitTest.java | 28 +- .../filesystem/HDFSCarbonFileTest.java | 3 +- .../filesystem/LocalCarbonFileTest.java | 20 + .../BooleanDataTypesInsertTest.scala| 5 ++-- .../carbondata/lcm/locks/LocalFileLockTest.java | 2 +- .../loading/csvinput/CSVInputFormatTest.java| 1 + 8 files changed, 64 insertions(+), 36 deletions(-) -- http://git-wip-us.apache.org/repos/asf/carbondata/blob/8a9dd8b2/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java -- diff --git a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java index 75ea074..cb80877 100644 --- a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java +++ b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java @@ -17,17 +17,20 @@ package org.apache.carbondata.core.locks; -import java.io.FileOutputStream; import java.io.IOException; import java.nio.channels.FileChannel; import java.nio.channels.FileLock; import java.nio.channels.OverlappingFileLockException; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.datastore.filesystem.CarbonFile; import org.apache.carbondata.core.datastore.impl.FileFactory; import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; +import org.apache.carbondata.core.util.CarbonUtil; /** * This class handles the file locking in the local file system. @@ -40,11 +43,6 @@ public class LocalFileLock extends AbstractCarbonLock { private String location; /** - * fileOutputStream of the local lock file - */ - private FileOutputStream fileOutputStream; - - /** * channel is the FileChannel of the lock file. */ private FileChannel channel; @@ -104,8 +102,8 @@ public class LocalFileLock extends AbstractCarbonLock { FileFactory.createNewLockFile(lockFilePath, FileFactory.getFileType(location)); } - fileOutputStream = new FileOutputStream(lockFilePath); - channel = fileOutputStream.getChannel(); + channel = FileChannel.open(Paths.get(lockFilePath), StandardOpenOption.WRITE, + StandardOpenOption.APPEND); try { fileLock = channel.tryLock(); } catch (OverlappingFileLockException e) { @@ -137,11 +135,17 @@ public class LocalFileLock extends AbstractCarbonLock { } catch (IOException e) { status = false; } finally { - if (null != fileOutputStream) { -try { - fileOutputStream.close(); -} catch (IOException e) { - LOGGER.error(e.getMessage()); + CarbonUtil.closeStreams(channel); + + // deleting the lock file after releasing the lock. + if (null != lockFilePath) { +CarbonFile lockFile = FileFactory.getCarbonFile(lockFilePath, +FileFactory.getFileType(lockFilePath)); +if (!lockFile.exists() || lockFile.delete()) { + LOGGER.info("Successfully deleted the lock file " + lockFilePath); +} else { + LOGGER.error("Not able to delete the lock file " + lockFilePath); + status = false; } } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/8a9dd8b2/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java -- diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java index da61a94..30144c1 100644 --- a/core/src/test/java/org
[46/49] carbondata git commit: [CARBONDATA-1114][Tests] Fix bugs in tests in windows env
[CARBONDATA-1114][Tests] Fix bugs in tests in windows env Fix bugs in tests that will cause failure under windows env This closes #1994 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/58c17539 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/58c17539 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/58c17539 Branch: refs/heads/carbonstore-rebase5 Commit: 58c17539b468e6ae089dfc51fbf3af85698047fc Parents: 2414fb0 Author: xuchuanyin Authored: Sat Feb 24 21:18:17 2018 +0800 Committer: Jacky Li Committed: Wed Feb 28 22:05:23 2018 +0800 -- .../carbondata/core/locks/LocalFileLock.java| 30 +++- .../store/impl/DFSFileReaderImplUnitTest.java | 11 --- .../store/impl/FileFactoryImplUnitTest.java | 28 +- .../filesystem/HDFSCarbonFileTest.java | 3 +- .../filesystem/LocalCarbonFileTest.java | 20 + .../BooleanDataTypesInsertTest.scala| 5 ++-- .../carbondata/lcm/locks/LocalFileLockTest.java | 2 +- .../loading/csvinput/CSVInputFormatTest.java| 1 + 8 files changed, 64 insertions(+), 36 deletions(-) -- http://git-wip-us.apache.org/repos/asf/carbondata/blob/58c17539/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java -- diff --git a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java index 75ea074..cb80877 100644 --- a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java +++ b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java @@ -17,17 +17,20 @@ package org.apache.carbondata.core.locks; -import java.io.FileOutputStream; import java.io.IOException; import java.nio.channels.FileChannel; import java.nio.channels.FileLock; import java.nio.channels.OverlappingFileLockException; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.datastore.filesystem.CarbonFile; import org.apache.carbondata.core.datastore.impl.FileFactory; import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; +import org.apache.carbondata.core.util.CarbonUtil; /** * This class handles the file locking in the local file system. @@ -40,11 +43,6 @@ public class LocalFileLock extends AbstractCarbonLock { private String location; /** - * fileOutputStream of the local lock file - */ - private FileOutputStream fileOutputStream; - - /** * channel is the FileChannel of the lock file. */ private FileChannel channel; @@ -104,8 +102,8 @@ public class LocalFileLock extends AbstractCarbonLock { FileFactory.createNewLockFile(lockFilePath, FileFactory.getFileType(location)); } - fileOutputStream = new FileOutputStream(lockFilePath); - channel = fileOutputStream.getChannel(); + channel = FileChannel.open(Paths.get(lockFilePath), StandardOpenOption.WRITE, + StandardOpenOption.APPEND); try { fileLock = channel.tryLock(); } catch (OverlappingFileLockException e) { @@ -137,11 +135,17 @@ public class LocalFileLock extends AbstractCarbonLock { } catch (IOException e) { status = false; } finally { - if (null != fileOutputStream) { -try { - fileOutputStream.close(); -} catch (IOException e) { - LOGGER.error(e.getMessage()); + CarbonUtil.closeStreams(channel); + + // deleting the lock file after releasing the lock. + if (null != lockFilePath) { +CarbonFile lockFile = FileFactory.getCarbonFile(lockFilePath, +FileFactory.getFileType(lockFilePath)); +if (!lockFile.exists() || lockFile.delete()) { + LOGGER.info("Successfully deleted the lock file " + lockFilePath); +} else { + LOGGER.error("Not able to delete the lock file " + lockFilePath); + status = false; } } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/58c17539/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java -- diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java index da61a94..30144c1 100644 --- a/core/src/test/java/or
[49/50] carbondata git commit: [CARBONDATA-1114][Tests] Fix bugs in tests in windows env
[CARBONDATA-1114][Tests] Fix bugs in tests in windows env Fix bugs in tests that will cause failure under windows env This closes #1994 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/c45a44d9 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/c45a44d9 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/c45a44d9 Branch: refs/heads/carbonstore-rebase4 Commit: c45a44d932fd565a39f9b128c0269c4e98d09b99 Parents: e5e187f Author: xuchuanyin Authored: Sat Feb 24 21:18:17 2018 +0800 Committer: Jacky Li Committed: Tue Feb 27 17:04:20 2018 +0800 -- .../carbondata/core/locks/LocalFileLock.java| 30 +++- .../store/impl/DFSFileReaderImplUnitTest.java | 11 --- .../store/impl/FileFactoryImplUnitTest.java | 28 +- .../filesystem/HDFSCarbonFileTest.java | 3 +- .../filesystem/LocalCarbonFileTest.java | 20 + .../BooleanDataTypesInsertTest.scala| 5 ++-- .../carbondata/lcm/locks/LocalFileLockTest.java | 2 +- .../loading/csvinput/CSVInputFormatTest.java| 1 + 8 files changed, 64 insertions(+), 36 deletions(-) -- http://git-wip-us.apache.org/repos/asf/carbondata/blob/c45a44d9/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java -- diff --git a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java index 75ea074..cb80877 100644 --- a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java +++ b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java @@ -17,17 +17,20 @@ package org.apache.carbondata.core.locks; -import java.io.FileOutputStream; import java.io.IOException; import java.nio.channels.FileChannel; import java.nio.channels.FileLock; import java.nio.channels.OverlappingFileLockException; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.datastore.filesystem.CarbonFile; import org.apache.carbondata.core.datastore.impl.FileFactory; import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; +import org.apache.carbondata.core.util.CarbonUtil; /** * This class handles the file locking in the local file system. @@ -40,11 +43,6 @@ public class LocalFileLock extends AbstractCarbonLock { private String location; /** - * fileOutputStream of the local lock file - */ - private FileOutputStream fileOutputStream; - - /** * channel is the FileChannel of the lock file. */ private FileChannel channel; @@ -104,8 +102,8 @@ public class LocalFileLock extends AbstractCarbonLock { FileFactory.createNewLockFile(lockFilePath, FileFactory.getFileType(location)); } - fileOutputStream = new FileOutputStream(lockFilePath); - channel = fileOutputStream.getChannel(); + channel = FileChannel.open(Paths.get(lockFilePath), StandardOpenOption.WRITE, + StandardOpenOption.APPEND); try { fileLock = channel.tryLock(); } catch (OverlappingFileLockException e) { @@ -137,11 +135,17 @@ public class LocalFileLock extends AbstractCarbonLock { } catch (IOException e) { status = false; } finally { - if (null != fileOutputStream) { -try { - fileOutputStream.close(); -} catch (IOException e) { - LOGGER.error(e.getMessage()); + CarbonUtil.closeStreams(channel); + + // deleting the lock file after releasing the lock. + if (null != lockFilePath) { +CarbonFile lockFile = FileFactory.getCarbonFile(lockFilePath, +FileFactory.getFileType(lockFilePath)); +if (!lockFile.exists() || lockFile.delete()) { + LOGGER.info("Successfully deleted the lock file " + lockFilePath); +} else { + LOGGER.error("Not able to delete the lock file " + lockFilePath); + status = false; } } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/c45a44d9/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java -- diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java index da61a94..30144c1 100644 --- a/core/src/test/java/or
[48/49] carbondata git commit: [CARBONDATA-1114][Tests] Fix bugs in tests in windows env
[CARBONDATA-1114][Tests] Fix bugs in tests in windows env Fix bugs in tests that will cause failure under windows env This closes #1994 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/de36a5d8 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/de36a5d8 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/de36a5d8 Branch: refs/heads/carbonstore-rebase4 Commit: de36a5d85d992dee4380ae228e6252baf1c31fe9 Parents: fa1a100 Author: xuchuanyin Authored: Sat Feb 24 21:18:17 2018 +0800 Committer: Jacky Li Committed: Tue Feb 27 09:24:02 2018 +0800 -- .../carbondata/core/locks/LocalFileLock.java| 30 +++- .../store/impl/DFSFileReaderImplUnitTest.java | 11 --- .../store/impl/FileFactoryImplUnitTest.java | 28 +- .../filesystem/HDFSCarbonFileTest.java | 3 +- .../filesystem/LocalCarbonFileTest.java | 20 + .../BooleanDataTypesInsertTest.scala| 5 ++-- .../carbondata/lcm/locks/LocalFileLockTest.java | 2 +- .../loading/csvinput/CSVInputFormatTest.java| 1 + 8 files changed, 64 insertions(+), 36 deletions(-) -- http://git-wip-us.apache.org/repos/asf/carbondata/blob/de36a5d8/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java -- diff --git a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java index 75ea074..cb80877 100644 --- a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java +++ b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java @@ -17,17 +17,20 @@ package org.apache.carbondata.core.locks; -import java.io.FileOutputStream; import java.io.IOException; import java.nio.channels.FileChannel; import java.nio.channels.FileLock; import java.nio.channels.OverlappingFileLockException; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.datastore.filesystem.CarbonFile; import org.apache.carbondata.core.datastore.impl.FileFactory; import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; +import org.apache.carbondata.core.util.CarbonUtil; /** * This class handles the file locking in the local file system. @@ -40,11 +43,6 @@ public class LocalFileLock extends AbstractCarbonLock { private String location; /** - * fileOutputStream of the local lock file - */ - private FileOutputStream fileOutputStream; - - /** * channel is the FileChannel of the lock file. */ private FileChannel channel; @@ -104,8 +102,8 @@ public class LocalFileLock extends AbstractCarbonLock { FileFactory.createNewLockFile(lockFilePath, FileFactory.getFileType(location)); } - fileOutputStream = new FileOutputStream(lockFilePath); - channel = fileOutputStream.getChannel(); + channel = FileChannel.open(Paths.get(lockFilePath), StandardOpenOption.WRITE, + StandardOpenOption.APPEND); try { fileLock = channel.tryLock(); } catch (OverlappingFileLockException e) { @@ -137,11 +135,17 @@ public class LocalFileLock extends AbstractCarbonLock { } catch (IOException e) { status = false; } finally { - if (null != fileOutputStream) { -try { - fileOutputStream.close(); -} catch (IOException e) { - LOGGER.error(e.getMessage()); + CarbonUtil.closeStreams(channel); + + // deleting the lock file after releasing the lock. + if (null != lockFilePath) { +CarbonFile lockFile = FileFactory.getCarbonFile(lockFilePath, +FileFactory.getFileType(lockFilePath)); +if (!lockFile.exists() || lockFile.delete()) { + LOGGER.info("Successfully deleted the lock file " + lockFilePath); +} else { + LOGGER.error("Not able to delete the lock file " + lockFilePath); + status = false; } } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/de36a5d8/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java -- diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java index da61a94..30144c1 100644 --- a/core/src/test/java/or
carbondata git commit: [CARBONDATA-1114][Tests] Fix bugs in tests in windows env
Repository: carbondata Updated Branches: refs/heads/carbonstore ab9b4cf89 -> 37080e9a3 [CARBONDATA-1114][Tests] Fix bugs in tests in windows env Fix bugs in tests that will cause failure under windows env This closes #1994 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/37080e9a Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/37080e9a Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/37080e9a Branch: refs/heads/carbonstore Commit: 37080e9a3eb9472265eea1b0387c3e4f3047619a Parents: ab9b4cf Author: xuchuanyin Authored: Sat Feb 24 21:18:17 2018 +0800 Committer: Jacky Li Committed: Mon Feb 26 23:30:00 2018 +0800 -- .../carbondata/core/locks/LocalFileLock.java| 38 +--- .../store/impl/DFSFileReaderImplUnitTest.java | 11 +++--- .../store/impl/FileFactoryImplUnitTest.java | 28 +++ .../filesystem/HDFSCarbonFileTest.java | 3 +- .../filesystem/LocalCarbonFileTest.java | 20 +++ .../BooleanDataTypesInsertTest.scala| 5 +-- .../carbondata/lcm/locks/LocalFileLockTest.java | 2 +- .../loading/csvinput/CSVInputFormatTest.java| 1 + 8 files changed, 63 insertions(+), 45 deletions(-) -- http://git-wip-us.apache.org/repos/asf/carbondata/blob/37080e9a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java -- diff --git a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java index c3dfb57..793d074 100644 --- a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java +++ b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java @@ -17,11 +17,12 @@ package org.apache.carbondata.core.locks; -import java.io.FileOutputStream; import java.io.IOException; import java.nio.channels.FileChannel; import java.nio.channels.FileLock; import java.nio.channels.OverlappingFileLockException; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; @@ -29,6 +30,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.datastore.filesystem.CarbonFile; import org.apache.carbondata.core.datastore.impl.FileFactory; import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; +import org.apache.carbondata.core.util.CarbonUtil; /** * This class handles the file locking in the local file system. @@ -41,11 +43,6 @@ public class LocalFileLock extends AbstractCarbonLock { private String location; /** - * fileOutputStream of the local lock file - */ - private FileOutputStream fileOutputStream; - - /** * channel is the FileChannel of the lock file. */ private FileChannel channel; @@ -105,8 +102,8 @@ public class LocalFileLock extends AbstractCarbonLock { FileFactory.createNewLockFile(lockFilePath, FileFactory.getFileType(location)); } - fileOutputStream = new FileOutputStream(lockFilePath); - channel = fileOutputStream.getChannel(); + channel = FileChannel.open(Paths.get(lockFilePath), StandardOpenOption.WRITE, + StandardOpenOption.APPEND); try { fileLock = channel.tryLock(); } catch (OverlappingFileLockException e) { @@ -139,20 +136,17 @@ public class LocalFileLock extends AbstractCarbonLock { } catch (IOException e) { status = false; } finally { - if (null != fileOutputStream) { -try { - fileOutputStream.close(); - // deleting the lock file after releasing the lock. - CarbonFile lockFile = FileFactory - .getCarbonFile(lockFilePath, FileFactory.getFileType(lockFilePath)); - if (!lockFile.exists() || lockFile.delete()) { -LOGGER.info("Successfully deleted the lock file " + lockFilePath); - } else { -LOGGER.error("Not able to delete the lock file " + lockFilePath); -status = false; - } -} catch (IOException e) { - LOGGER.error(e.getMessage()); + CarbonUtil.closeStreams(channel); + + // deleting the lock file after releasing the lock. + if (null != lockFilePath) { +CarbonFile lockFile = FileFactory.getCarbonFile(lockFilePath, +FileFactory.getFileType(lockFilePath)); +if (!lockFile.exists() || lockFile.delete()) { + LOGGER.info("Successfully deleted the lock file " + lockFilePath); +} else { + LOGGER.error("Not able to delete the lock file " + lockFilePath); + status = false; } } }