[Review][CARBONDATA-1386] fixed findbugs errors in carbondata-core fixed findbugs errors in carbondata-core
This closes #1263 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/500654e6 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/500654e6 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/500654e6 Branch: refs/heads/master Commit: 500654e60d4fb7b1ab5eda39f29a415933838006 Parents: 2ee7775 Author: sraghunandan <[email protected]> Authored: Thu Aug 17 19:14:48 2017 +0530 Committer: Jacky Li <[email protected]> Committed: Sat Aug 19 09:52:52 2017 +0800 ---------------------------------------------------------------------- .../impl/ExtendedRollingFileAppender.java | 4 + .../dictionary/AbstractDictionaryCache.java | 2 +- .../dictionary/DoubleArrayTrieDictionary.java | 12 +- .../dictionary/ReverseDictionaryCache.java | 2 +- .../core/constants/CarbonCommonConstants.java | 3 +- .../core/datamap/DataMapStoreManager.java | 4 +- .../carbondata/core/datamap/TableDataMap.java | 6 +- .../datastore/AbstractBlockIndexStoreCache.java | 8 +- .../core/datastore/BlockIndexStore.java | 2 +- .../core/datastore/block/TableBlockInfo.java | 12 +- .../core/datastore/block/TableTaskInfo.java | 23 ++++ .../impl/FixedLengthDimensionDataChunk.java | 8 ++ .../UnsafeAbstractDimensionDataChunkStore.java | 10 +- ...nsafeFixedLengthDimensionDataChunkStore.java | 24 ++-- ...afeVariableLengthDimesionDataChunkStore.java | 28 ++--- .../columnar/BlockIndexerStorageForShort.java | 2 +- .../core/datastore/impl/FileFactory.java | 3 + .../core/datastore/impl/FileHolderImpl.java | 8 +- .../core/datastore/page/ColumnPage.java | 2 + .../page/UnsafeFixLengthColumnPage.java | 87 +++++++------ .../page/UnsafeVarLengthColumnPage.java | 10 +- .../page/encoding/DeltaIntegralCodec.java | 9 +- .../core/datastore/page/encoding/RLECodec.java | 2 + .../statistics/PrimitivePageStatsCollector.java | 13 ++ .../core/datastore/row/LoadStatusType.java | 4 - .../client/DictionaryClientHandler.java | 2 +- .../generator/ServerDictionaryGenerator.java | 11 +- .../generator/TableDictionaryGenerator.java | 35 +++--- .../dictionary/server/DictionaryServer.java | 2 +- .../carbondata/core/indexstore/Blocklet.java | 11 +- .../core/indexstore/UnsafeMemoryDMStore.java | 39 +++--- .../blockletindex/BlockletDataMap.java | 19 +-- .../BlockletDataRefNodeWrapper.java | 2 +- .../core/indexstore/row/UnsafeDataMapRow.java | 25 ++-- .../MultiDimKeyVarLengthEquiSplitGenerator.java | 23 ---- ...tiDimKeyVarLengthVariableSplitGenerator.java | 23 ---- .../DateDirectDictionaryGenerator.java | 2 +- .../TimeStampDirectDictionaryGenerator.java | 2 +- .../carbondata/core/locks/HdfsFileLock.java | 2 +- .../carbondata/core/locks/ZooKeeperLocking.java | 12 +- .../carbondata/core/memory/CarbonUnsafe.java | 6 +- .../core/memory/IntPointerBuffer.java | 4 +- .../carbondata/core/memory/MemoryBlock.java | 7 -- .../core/memory/UnsafeMemoryAllocator.java | 6 +- .../core/memory/UnsafeMemoryManager.java | 12 +- .../core/memory/UnsafeSortMemoryManager.java | 12 +- .../core/metadata/ColumnPageCodecMeta.java | 2 + .../datatype/DecimalConverterFactory.java | 6 - .../core/metadata/schema/table/TableInfo.java | 10 +- .../core/mutate/CarbonUpdateUtil.java | 4 - .../RestructureBasedRawResultCollector.java | 3 +- .../scan/complextypes/ComplexQueryType.java | 3 - .../core/scan/executor/util/QueryUtil.java | 3 +- .../scan/executor/util/RestructureUtil.java | 4 +- .../expression/RangeExpressionEvaluator.java | 20 +-- .../expression/conditional/ListExpression.java | 11 +- .../scan/filter/FilterExpressionProcessor.java | 15 ++- .../carbondata/core/scan/filter/FilterUtil.java | 8 +- .../RestructureExcludeFilterExecutorImpl.java | 12 +- .../RestructureIncludeFilterExecutorImpl.java | 13 +- .../executer/RowLevelFilterExecuterImpl.java | 4 +- .../scan/filter/intf/FilterOptimizerImpl.java | 4 +- .../filter/optimizer/RangeFilterOptmizer.java | 2 +- .../MeasureColumnResolvedFilterInfo.java | 6 - .../visitor/CustomTypeDictionaryVisitor.java | 39 +++--- .../visitor/DictionaryColumnVisitor.java | 48 +++---- .../visitor/MeasureColumnVisitor.java | 50 ++++---- .../visitor/NoDictionaryTypeVisitor.java | 48 +++---- .../visitor/RangeDictionaryColumnVisitor.java | 28 +++-- .../visitor/RangeDirectDictionaryVisitor.java | 53 ++++---- .../visitor/RangeNoDictionaryTypeVisitor.java | 52 ++++---- .../scan/result/vector/ColumnVectorInfo.java | 23 ++++ .../core/stats/QueryStatisticsRecorderImpl.java | 14 ++- .../apache/carbondata/core/util/ByteUtil.java | 49 ++++---- .../carbondata/core/util/CarbonProperties.java | 28 ++--- .../apache/carbondata/core/util/CarbonUtil.java | 11 +- .../core/util/DataTypeConverterImpl.java | 2 +- .../carbondata/core/util/DataTypeUtil.java | 6 +- .../core/util/comparator/Comparator.java | 8 +- dev/findbugs-exclude.xml | 124 ++++--------------- pom.xml | 2 +- .../sort/unsafe/UnsafeCarbonRowPage.java | 72 +++++------ .../newflow/sort/unsafe/UnsafeSortDataRows.java | 2 +- .../unsafe/comparator/UnsafeRowComparator.java | 24 ++-- .../UnsafeRowComparatorForNormalDIms.java | 4 +- 85 files changed, 663 insertions(+), 699 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/common/src/main/java/org/apache/carbondata/common/logging/impl/ExtendedRollingFileAppender.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/carbondata/common/logging/impl/ExtendedRollingFileAppender.java b/common/src/main/java/org/apache/carbondata/common/logging/impl/ExtendedRollingFileAppender.java index 1261726..828dd14 100644 --- a/common/src/main/java/org/apache/carbondata/common/logging/impl/ExtendedRollingFileAppender.java +++ b/common/src/main/java/org/apache/carbondata/common/logging/impl/ExtendedRollingFileAppender.java @@ -74,6 +74,10 @@ public class ExtendedRollingFileAppender extends RollingFileAppender { } }); + if (null == files) { + return; + } + int backupFiles = files.length - 1; if (backupFiles <= maxBackupIndex) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java index f0b8c78..6910888 100644 --- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java +++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java @@ -173,7 +173,7 @@ public abstract class AbstractDictionaryCache<K extends DictionaryColumnUniqueId protected long getSortIndexSize(long numOfRecords) { // sort index has sort index and reverse sort index,each is 4 byte integer. // 32 byte is the array header of both the integer arrays - return numOfRecords * ObjectSizeCalculator.estimate(new Integer(0), 16) * 2 + 32; + return numOfRecords * ObjectSizeCalculator.estimate(0, 16) * 2 + 32; } /** http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java index cab1926..ef36d7a 100644 --- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java +++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java @@ -35,10 +35,10 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants; */ public class DoubleArrayTrieDictionary { - public static final byte[] HEAD_MAGIC = new byte[]{ + private static final byte[] HEAD_MAGIC = new byte[]{ 0x44, 0x41, 0x54, 0x54, 0x72, 0x69, 0x65, 0x44, 0x69, 0x63, 0x74 }; // "DATTrieDict" - public static final int HEAD_LEN = HEAD_MAGIC.length; + private static final int HEAD_LEN = HEAD_MAGIC.length; private static final int INIT_CAPA_VALUE = 256; // init len of double array private static final int BASE_ROOT_VALUE = 1; // root base value of trie root @@ -168,7 +168,7 @@ public class DoubleArrayTrieDictionary { return null; } if (check[cpos] == pos) { - children.add(new Integer(i)); + children.add(i); } } return children; @@ -227,9 +227,9 @@ public class DoubleArrayTrieDictionary { private int conflict(int start, int bKey) { int from = start; TreeSet<Integer> children = getChildren(from); - children.add(new Integer(bKey)); + children.add(bKey); int newBasePos = findFreeRoom(children); - children.remove(new Integer(bKey)); + children.remove(bKey); int oldBasePos = base[start]; base[start] = newBasePos; @@ -351,7 +351,7 @@ public class DoubleArrayTrieDictionary { */ public void read(DataInputStream in) throws IOException { byte[] header = new byte[HEAD_LEN]; - in.read(header); + in.readFully(header); int comp = 0; for (int i = 0; i < HEAD_LEN; i++) { comp = HEAD_MAGIC[i] - header[i]; http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCache.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCache.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCache.java index 28568b5..fd5a954 100644 --- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCache.java +++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCache.java @@ -58,7 +58,7 @@ public class ReverseDictionaryCache<K extends DictionaryColumnUniqueIdentifier, private static final long sizeOfHashMapNode = ObjectSizeCalculator.estimate(new DictionaryByteArrayWrapper(new byte[0]), 16) + - ObjectSizeCalculator.estimate(new Integer(0), 16); + ObjectSizeCalculator.estimate(0, 16); private static final long byteArraySize = ObjectSizeCalculator.estimate(new byte[0], 16); http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java index 8939a7e..944abc4 100644 --- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java +++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java @@ -189,8 +189,7 @@ public final class CarbonCommonConstants { /** * Bytes for string 0, it is used in codegen in case of null values. */ - public static final byte[] ZERO_BYTE_ARRAY = "0".getBytes(); - + public static final byte[] ZERO_BYTE_ARRAY = "0".getBytes(Charset.forName(DEFAULT_CHARSET)); /** * FILE STATUS IN-PROGRESS */ http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java index f5bc22f..54318b5 100644 --- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java +++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java @@ -92,7 +92,7 @@ public final class DataMapStoreManager { try { DataMapFactory dataMapFactory = factoryClass.newInstance(); dataMapFactory.init(identifier, dataMapName); - dataMap = new TableDataMap(identifier, dataMapName, dataMapFactory); + dataMap = new TableDataMap(dataMapName, dataMapFactory); } catch (Exception e) { LOGGER.error(e); throw new RuntimeException(e); @@ -119,7 +119,7 @@ public final class DataMapStoreManager { * @param dataMapName */ public void clearDataMap(AbsoluteTableIdentifier identifier, String dataMapName) { - List<TableDataMap> tableDataMaps = allDataMaps.get(identifier); + List<TableDataMap> tableDataMaps = allDataMaps.get(identifier.uniqueName()); if (tableDataMaps != null) { int i = 0; for (TableDataMap tableDataMap: tableDataMaps) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java index b55c5d9..5571538 100644 --- a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java +++ b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java @@ -25,7 +25,6 @@ import org.apache.carbondata.core.datamap.dev.DataMapFactory; import org.apache.carbondata.core.events.ChangeEvent; import org.apache.carbondata.core.events.EventListener; import org.apache.carbondata.core.indexstore.Blocklet; -import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf; /** @@ -34,8 +33,6 @@ import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf; */ public final class TableDataMap implements EventListener { - private AbsoluteTableIdentifier identifier; - private String dataMapName; private DataMapFactory dataMapFactory; @@ -43,9 +40,8 @@ public final class TableDataMap implements EventListener { /** * It is called to initialize and load the required table datamap metadata. */ - public TableDataMap(AbsoluteTableIdentifier identifier, String dataMapName, + public TableDataMap(String dataMapName, DataMapFactory dataMapFactory) { - this.identifier = identifier; this.dataMapName = dataMapName; this.dataMapFactory = dataMapFactory; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/AbstractBlockIndexStoreCache.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/AbstractBlockIndexStoreCache.java b/core/src/main/java/org/apache/carbondata/core/datastore/AbstractBlockIndexStoreCache.java index 8cfc602..3a62783 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/AbstractBlockIndexStoreCache.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/AbstractBlockIndexStoreCache.java @@ -41,10 +41,7 @@ import org.apache.carbondata.core.util.ObjectSizeCalculator; */ public abstract class AbstractBlockIndexStoreCache<K, V> implements Cache<TableBlockUniqueIdentifier, AbstractIndex> { - /** - * carbon store path - */ - protected String carbonStorePath; + /** * CarbonLRU cache */ @@ -70,8 +67,7 @@ public abstract class AbstractBlockIndexStoreCache<K, V> */ protected Map<String, Object> segmentIDLock; - public AbstractBlockIndexStoreCache(String carbonStorePath, CarbonLRUCache lruCache) { - this.carbonStorePath = carbonStorePath; + public AbstractBlockIndexStoreCache(CarbonLRUCache lruCache) { this.lruCache = lruCache; blockInfoLock = new ConcurrentHashMap<BlockInfo, Object>(); segmentIDLock = new ConcurrentHashMap<String, Object>(); http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/BlockIndexStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/BlockIndexStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/BlockIndexStore.java index 32ca096..fad6ed6 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/BlockIndexStore.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/BlockIndexStore.java @@ -59,7 +59,7 @@ public class BlockIndexStore<K, V> extends AbstractBlockIndexStoreCache<K, V> { private static final LogService LOGGER = LogServiceFactory.getLogService(BlockIndexStore.class.getName()); public BlockIndexStore(String carbonStorePath, CarbonLRUCache lruCache) { - super(carbonStorePath, lruCache); + super(lruCache); } /** http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java index 316e202..6389528 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java @@ -190,13 +190,15 @@ public class TableBlockInfo implements Distributable, Serializable { if (blockLength != other.blockLength) { return false; } - if (filePath == null && other.filePath != null) { - return false; - } else if (filePath != null && other.filePath == null) { - return false; - } else if (!filePath.equals(other.filePath)) { + + if (null == filePath || null == other.filePath) { + return false; + } + + if (!filePath.equals(other.filePath)) { return false; } + if (blockletInfos.getStartBlockletNumber() != other.blockletInfos.getStartBlockletNumber()) { return false; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/block/TableTaskInfo.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableTaskInfo.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableTaskInfo.java index 73aa810..89a7551 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableTaskInfo.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableTaskInfo.java @@ -60,6 +60,29 @@ public class TableTaskInfo implements Distributable { return taskId.compareTo(((TableTaskInfo)o).getTaskId()); } + @Override public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (null == obj) { + return false; + } + + if (!(obj instanceof TableTaskInfo)) { + return false; + } + + return 0 == taskId.compareTo(((TableTaskInfo)obj).getTaskId()); + } + + @Override public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((taskId == null) ? 0 : taskId.hashCode()); + return result; + } + /** * Finding which node has the maximum number of blocks for it. * @param blockList http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java index bb76bbf..97fd226 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java @@ -110,6 +110,10 @@ public class FixedLengthDimensionDataChunk extends AbstractDimensionDataChunk { case LONG: vector.putLong(vectorOffset++, (long) valueFromSurrogate); break; + default: + throw new IllegalArgumentException( + "unsupported data type: " + columnVectorInfo.directDictionaryGenerator + .getReturnType()); } } } @@ -150,6 +154,10 @@ public class FixedLengthDimensionDataChunk extends AbstractDimensionDataChunk { case LONG: vector.putLong(vectorOffset++, (long) valueFromSurrogate); break; + default: + throw new IllegalArgumentException( + "unsupported data type: " + columnVectorInfo.directDictionaryGenerator + .getReturnType()); } } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java index 3752fb6..704f2d3 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java @@ -92,16 +92,16 @@ public abstract class UnsafeAbstractDimensionDataChunkStore implements Dimension invertedIndex.length * CarbonCommonConstants.INT_SIZE_IN_BYTE; } // copy the data to memory - CarbonUnsafe.unsafe + CarbonUnsafe.getUnsafe() .copyMemory(data, CarbonUnsafe.BYTE_ARRAY_OFFSET, dataPageMemoryBlock.getBaseObject(), dataPageMemoryBlock.getBaseOffset(), this.dataLength); // if inverted index is present then copy the inverted index // and reverse inverted index to memory if (isExplicitSorted) { - CarbonUnsafe.unsafe.copyMemory(invertedIndex, CarbonUnsafe.INT_ARRAY_OFFSET, + CarbonUnsafe.getUnsafe().copyMemory(invertedIndex, CarbonUnsafe.INT_ARRAY_OFFSET, dataPageMemoryBlock.getBaseObject(), dataPageMemoryBlock.getBaseOffset() + dataLength, invertedIndex.length * CarbonCommonConstants.INT_SIZE_IN_BYTE); - CarbonUnsafe.unsafe.copyMemory(invertedIndexReverse, CarbonUnsafe.INT_ARRAY_OFFSET, + CarbonUnsafe.getUnsafe().copyMemory(invertedIndexReverse, CarbonUnsafe.INT_ARRAY_OFFSET, dataPageMemoryBlock.getBaseObject(), dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset, invertedIndexReverse.length * CarbonCommonConstants.INT_SIZE_IN_BYTE); @@ -129,8 +129,8 @@ public abstract class UnsafeAbstractDimensionDataChunkStore implements Dimension * @return inverted index based on row id passed */ @Override public int getInvertedIndex(int rowId) { - return CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(), - dataPageMemoryBlock.getBaseOffset() + dataLength + (rowId + return CarbonUnsafe.getUnsafe().getInt(dataPageMemoryBlock.getBaseObject(), + dataPageMemoryBlock.getBaseOffset() + dataLength + ((long)rowId * CarbonCommonConstants.INT_SIZE_IN_BYTE)); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java index 4ff9c77..8c8d08f 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java @@ -53,16 +53,16 @@ public class UnsafeFixedLengthDimensionDataChunkStore @Override public byte[] getRow(int rowId) { // if column was explicitly sorted we need to get the rowid based inverted index reverse if (isExplicitSorted) { - rowId = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(), - dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset + (rowId + rowId = CarbonUnsafe.getUnsafe().getInt(dataPageMemoryBlock.getBaseObject(), + dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset + ((long)rowId * CarbonCommonConstants.INT_SIZE_IN_BYTE)); } // creating a row byte[] data = new byte[columnValueSize]; //copy the row from memory block based on offset // offset position will be index * each column value length - CarbonUnsafe.unsafe.copyMemory(dataPageMemoryBlock.getBaseObject(), - dataPageMemoryBlock.getBaseOffset() + (rowId * columnValueSize), data, + CarbonUnsafe.getUnsafe().copyMemory(dataPageMemoryBlock.getBaseObject(), + dataPageMemoryBlock.getBaseOffset() + ((long)rowId * columnValueSize), data, CarbonUnsafe.BYTE_ARRAY_OFFSET, columnValueSize); return data; } @@ -77,8 +77,8 @@ public class UnsafeFixedLengthDimensionDataChunkStore @Override public int getSurrogate(int index) { // if column was explicitly sorted we need to get the rowid based inverted index reverse if (isExplicitSorted) { - index = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(), - dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset + (index + index = CarbonUnsafe.getUnsafe().getInt(dataPageMemoryBlock.getBaseObject(), + dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset + ((long)index * CarbonCommonConstants.INT_SIZE_IN_BYTE)); } // below part is to convert the byte array to surrogate value @@ -86,7 +86,7 @@ public class UnsafeFixedLengthDimensionDataChunkStore int surrogate = 0; for (int i = 0; i < columnValueSize; i++) { surrogate <<= 8; - surrogate ^= CarbonUnsafe.unsafe.getByte(dataPageMemoryBlock.getBaseObject(), + surrogate ^= CarbonUnsafe.getUnsafe().getByte(dataPageMemoryBlock.getBaseObject(), dataPageMemoryBlock.getBaseOffset() + startOffsetOfData) & 0xFF; startOffsetOfData++; } @@ -103,14 +103,14 @@ public class UnsafeFixedLengthDimensionDataChunkStore @Override public void fillRow(int rowId, byte[] buffer, int offset) { // if column was explicitly sorted we need to get the rowid based inverted index reverse if (isExplicitSorted) { - rowId = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(), - dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset + (rowId + rowId = CarbonUnsafe.getUnsafe().getInt(dataPageMemoryBlock.getBaseObject(), + dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset + ((long)rowId * CarbonCommonConstants.INT_SIZE_IN_BYTE)); } //copy the row from memory block based on offset // offset position will be index * each column value length - CarbonUnsafe.unsafe.copyMemory(dataPageMemoryBlock.getBaseObject(), - dataPageMemoryBlock.getBaseOffset() + (rowId * columnValueSize), buffer, + CarbonUnsafe.getUnsafe().copyMemory(dataPageMemoryBlock.getBaseObject(), + dataPageMemoryBlock.getBaseOffset() + ((long)rowId * columnValueSize), buffer, CarbonUnsafe.BYTE_ARRAY_OFFSET + offset, columnValueSize); } @@ -133,7 +133,7 @@ public class UnsafeFixedLengthDimensionDataChunkStore index = index * columnValueSize; int compareResult = 0; for (int i = 0; i < compareValue.length; i++) { - compareResult = (CarbonUnsafe.unsafe + compareResult = (CarbonUnsafe.getUnsafe() .getByte(dataPageMemoryBlock.getBaseObject(), dataPageMemoryBlock.getBaseOffset() + index) & 0xff) - (compareValue[i] & 0xff); if (compareResult != 0) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java index c6c98f9..6193804 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java @@ -68,7 +68,7 @@ public class UnsafeVariableLengthDimesionDataChunkStore // position from where offsets will start this.dataPointersOffsets = this.invertedIndexReverseOffset; if (isExplicitSorted) { - this.dataPointersOffsets += numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE; + this.dataPointersOffsets += (long)numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE; } // As data is of variable length and data format is // <length in short><data><length in short><data> @@ -87,7 +87,7 @@ public class UnsafeVariableLengthDimesionDataChunkStore // as first position will be start from 2 byte as data is stored first in the memory block // we need to skip first two bytes this is because first two bytes will be length of the data // which we have to skip - CarbonUnsafe.unsafe.putInt(dataPageMemoryBlock.getBaseObject(), + CarbonUnsafe.getUnsafe().putInt(dataPageMemoryBlock.getBaseObject(), dataPageMemoryBlock.getBaseOffset() + pointerOffsets, CarbonCommonConstants.SHORT_SIZE_IN_BYTE); // incrementing the pointers as first value is already filled and as we are storing as int @@ -101,7 +101,7 @@ public class UnsafeVariableLengthDimesionDataChunkStore // as first offset is already stored, we need to start from the 2nd row in data array for (int i = 1; i < numberOfRows; i++) { // first copy the length of previous row - CarbonUnsafe.unsafe.copyMemory(dataPageMemoryBlock.getBaseObject(), + CarbonUnsafe.getUnsafe().copyMemory(dataPageMemoryBlock.getBaseObject(), dataPageMemoryBlock.getBaseOffset() + startOffset, length, CarbonUnsafe.BYTE_ARRAY_OFFSET, CarbonCommonConstants.SHORT_SIZE_IN_BYTE); buffer.put(length); @@ -114,7 +114,7 @@ public class UnsafeVariableLengthDimesionDataChunkStore buffer.clear(); // now put the offset of current row, here we need to add 2 more bytes as current will // also have length part so we have to skip length - CarbonUnsafe.unsafe.putInt(dataPageMemoryBlock.getBaseObject(), + CarbonUnsafe.getUnsafe().putInt(dataPageMemoryBlock.getBaseObject(), dataPageMemoryBlock.getBaseOffset() + pointerOffsets, startOffset + CarbonCommonConstants.SHORT_SIZE_IN_BYTE); // incrementing the pointers as first value is already filled and as we are storing as int @@ -133,8 +133,8 @@ public class UnsafeVariableLengthDimesionDataChunkStore @Override public byte[] getRow(int rowId) { // if column was explicitly sorted we need to get the rowid based inverted index reverse if (isExplicitSorted) { - rowId = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(), - dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset + (rowId + rowId = CarbonUnsafe.getUnsafe().getInt(dataPageMemoryBlock.getBaseObject(), + dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset + ((long)rowId * CarbonCommonConstants.INT_SIZE_IN_BYTE)); } // now to get the row from memory block we need to do following thing @@ -143,13 +143,13 @@ public class UnsafeVariableLengthDimesionDataChunkStore // Subtract the current row offset + 2 bytes(to skip the data length) with next row offset // else subtract the current row offset + 2 bytes(to skip the data length) // with complete data length - int currentDataOffset = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(), + int currentDataOffset = CarbonUnsafe.getUnsafe().getInt(dataPageMemoryBlock.getBaseObject(), dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets + (rowId * CarbonCommonConstants.INT_SIZE_IN_BYTE)); short length = 0; // calculating the length of data if (rowId < numberOfRows - 1) { - int OffsetOfNextdata = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(), + int OffsetOfNextdata = CarbonUnsafe.getUnsafe().getInt(dataPageMemoryBlock.getBaseObject(), dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets + ((rowId + 1) * CarbonCommonConstants.INT_SIZE_IN_BYTE)); length = (short) (OffsetOfNextdata - (currentDataOffset @@ -159,7 +159,7 @@ public class UnsafeVariableLengthDimesionDataChunkStore length = (short) (this.dataLength - currentDataOffset); } byte[] data = new byte[length]; - CarbonUnsafe.unsafe.copyMemory(dataPageMemoryBlock.getBaseObject(), + CarbonUnsafe.getUnsafe().copyMemory(dataPageMemoryBlock.getBaseObject(), dataPageMemoryBlock.getBaseOffset() + currentDataOffset, data, CarbonUnsafe.BYTE_ARRAY_OFFSET, length); return data; @@ -200,13 +200,13 @@ public class UnsafeVariableLengthDimesionDataChunkStore // Subtract the current row offset + 2 bytes(to skip the data length) with next row offset // else subtract the current row offset // with complete data length get the offset of set of data - int currentDataOffset = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(), - dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets + (index - * CarbonCommonConstants.INT_SIZE_IN_BYTE)); + int currentDataOffset = CarbonUnsafe.getUnsafe().getInt(dataPageMemoryBlock.getBaseObject(), + dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets + ((long)index + * CarbonCommonConstants.INT_SIZE_IN_BYTE * 1L)); short length = 0; // calculating the length of data if (index < numberOfRows - 1) { - int OffsetOfNextdata = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(), + int OffsetOfNextdata = CarbonUnsafe.getUnsafe().getInt(dataPageMemoryBlock.getBaseObject(), dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets + ((index + 1) * CarbonCommonConstants.INT_SIZE_IN_BYTE)); length = (short) (OffsetOfNextdata - (currentDataOffset @@ -220,7 +220,7 @@ public class UnsafeVariableLengthDimesionDataChunkStore int compareResult; int compareLength = Math.min(length , compareValue.length); for (int i = 0; i < compareLength; i++) { - compareResult = (CarbonUnsafe.unsafe.getByte(dataPageMemoryBlock.getBaseObject(), + compareResult = (CarbonUnsafe.getUnsafe().getByte(dataPageMemoryBlock.getBaseObject(), dataPageMemoryBlock.getBaseOffset() + currentDataOffset) & 0xff) - (compareValue[i] & 0xff); // if compare result is not equal we can break http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForShort.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForShort.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForShort.java index 7303b66..a91d6bc 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForShort.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForShort.java @@ -117,7 +117,7 @@ public class BlockIndexerStorageForShort implements IndexStorage<short[]> { } else { list.add(rowIds[i - 1]); } - double compressionPercentage = (((list.size() + map.size()) * 100) / rowIds.length); + int compressionPercentage = (((list.size() + map.size()) * 100) / rowIds.length); if (compressionPercentage > 70) { rowIdPage = rowIds; } else { http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java index 3477834..3191200 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java @@ -367,6 +367,9 @@ public final class FileFactory { return path.delete(); } File[] files = path.listFiles(); + if (null == files) { + return true; + } for (int i = 0; i < files.length; i++) { deleteAllFilesOfDir(files[i]); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileHolderImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileHolderImpl.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileHolderImpl.java index 8e6aafc..c867f75 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileHolderImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileHolderImpl.java @@ -217,7 +217,13 @@ public class FileHolderImpl implements FileHolder { @Override public DataInputStream getDataInputStream(String filePath, long offset) throws IOException { FileInputStream stream = new FileInputStream(filePath); - stream.skip(offset); + long skipped = stream.skip(offset); + long toSkip = offset - skipped; + while (toSkip <= 0) { + skipped = stream.skip(toSkip); + toSkip = toSkip - skipped; + } + return new DataInputStream(new BufferedInputStream(stream)); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java index a0eeb44..8b44f07 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java @@ -410,6 +410,8 @@ public abstract class ColumnPage { case DECIMAL: putDecimal(rowId, BigDecimal.ZERO); break; + default: + throw new IllegalArgumentException("unsupported data type: " + dataType); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java index f019ae8..06d952d 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java @@ -82,40 +82,40 @@ public class UnsafeFixLengthColumnPage extends ColumnPage { @Override public void putByte(int rowId, byte value) { long offset = rowId << byteBits; - CarbonUnsafe.unsafe.putByte(baseAddress, baseOffset + offset, value); + CarbonUnsafe.getUnsafe().putByte(baseAddress, baseOffset + offset, value); } @Override public void putShort(int rowId, short value) { long offset = rowId << shortBits; - CarbonUnsafe.unsafe.putShort(baseAddress, baseOffset + offset, value); + CarbonUnsafe.getUnsafe().putShort(baseAddress, baseOffset + offset, value); } @Override public void putShortInt(int rowId, int value) { byte[] data = ByteUtil.to3Bytes(value); - long offset = rowId * 3; - CarbonUnsafe.unsafe.putByte(baseAddress, baseOffset + offset, data[0]); - CarbonUnsafe.unsafe.putByte(baseAddress, baseOffset + offset + 1, data[1]); - CarbonUnsafe.unsafe.putByte(baseAddress, baseOffset + offset + 2, data[2]); + long offset = rowId * 3L; + CarbonUnsafe.getUnsafe().putByte(baseAddress, baseOffset + offset, data[0]); + CarbonUnsafe.getUnsafe().putByte(baseAddress, baseOffset + offset + 1, data[1]); + CarbonUnsafe.getUnsafe().putByte(baseAddress, baseOffset + offset + 2, data[2]); } @Override public void putInt(int rowId, int value) { long offset = rowId << intBits; - CarbonUnsafe.unsafe.putInt(baseAddress, baseOffset + offset, value); + CarbonUnsafe.getUnsafe().putInt(baseAddress, baseOffset + offset, value); } @Override public void putLong(int rowId, long value) { long offset = rowId << longBits; - CarbonUnsafe.unsafe.putLong(baseAddress, baseOffset + offset, value); + CarbonUnsafe.getUnsafe().putLong(baseAddress, baseOffset + offset, value); } @Override public void putDouble(int rowId, double value) { long offset = rowId << doubleBits; - CarbonUnsafe.unsafe.putDouble(baseAddress, baseOffset + offset, value); + CarbonUnsafe.getUnsafe().putDouble(baseAddress, baseOffset + offset, value); } @Override @@ -135,47 +135,47 @@ public class UnsafeFixLengthColumnPage extends ColumnPage { @Override public byte getByte(int rowId) { long offset = rowId << byteBits; - return CarbonUnsafe.unsafe.getByte(baseAddress, baseOffset + offset); + return CarbonUnsafe.getUnsafe().getByte(baseAddress, baseOffset + offset); } @Override public short getShort(int rowId) { long offset = rowId << shortBits; - return CarbonUnsafe.unsafe.getShort(baseAddress, baseOffset + offset); + return CarbonUnsafe.getUnsafe().getShort(baseAddress, baseOffset + offset); } @Override public int getShortInt(int rowId) { - long offset = rowId * 3; + long offset = rowId * 3L; byte[] data = new byte[3]; - data[0] = CarbonUnsafe.unsafe.getByte(baseAddress, baseOffset + offset); - data[1] = CarbonUnsafe.unsafe.getByte(baseAddress, baseOffset + offset + 1); - data[2] = CarbonUnsafe.unsafe.getByte(baseAddress, baseOffset + offset + 2); + data[0] = CarbonUnsafe.getUnsafe().getByte(baseAddress, baseOffset + offset); + data[1] = CarbonUnsafe.getUnsafe().getByte(baseAddress, baseOffset + offset + 1); + data[2] = CarbonUnsafe.getUnsafe().getByte(baseAddress, baseOffset + offset + 2); return ByteUtil.valueOf3Bytes(data, 0); } @Override public int getInt(int rowId) { long offset = rowId << intBits; - return CarbonUnsafe.unsafe.getInt(baseAddress, baseOffset + offset); + return CarbonUnsafe.getUnsafe().getInt(baseAddress, baseOffset + offset); } @Override public long getLong(int rowId) { long offset = rowId << longBits; - return CarbonUnsafe.unsafe.getLong(baseAddress, baseOffset + offset); + return CarbonUnsafe.getUnsafe().getLong(baseAddress, baseOffset + offset); } @Override public float getFloat(int rowId) { long offset = rowId << floatBits; - return CarbonUnsafe.unsafe.getFloat(baseAddress, baseOffset + offset); + return CarbonUnsafe.getUnsafe().getFloat(baseAddress, baseOffset + offset); } @Override public double getDouble(int rowId) { long offset = rowId << doubleBits; - return CarbonUnsafe.unsafe.getDouble(baseAddress, baseOffset + offset); + return CarbonUnsafe.getUnsafe().getDouble(baseAddress, baseOffset + offset); } @Override @@ -192,7 +192,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage { byte[] data = new byte[getPageSize()]; for (int i = 0; i < data.length; i++) { long offset = i << byteBits; - data[i] = CarbonUnsafe.unsafe.getByte(baseAddress, baseOffset + offset); + data[i] = CarbonUnsafe.getUnsafe().getByte(baseAddress, baseOffset + offset); } return data; } @@ -202,7 +202,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage { short[] data = new short[getPageSize()]; for (int i = 0; i < data.length; i++) { long offset = i << shortBits; - data[i] = CarbonUnsafe.unsafe.getShort(baseAddress, baseOffset + offset); + data[i] = CarbonUnsafe.getUnsafe().getShort(baseAddress, baseOffset + offset); } return data; } @@ -210,7 +210,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage { @Override public byte[] getShortIntPage() { byte[] data = new byte[pageSize * 3]; - CarbonUnsafe.unsafe.copyMemory(baseAddress, baseOffset, + CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset, data, CarbonUnsafe.BYTE_ARRAY_OFFSET, data.length); return data; } @@ -220,7 +220,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage { int[] data = new int[getPageSize()]; for (int i = 0; i < data.length; i++) { long offset = i << intBits; - data[i] = CarbonUnsafe.unsafe.getInt(baseAddress, baseOffset + offset); + data[i] = CarbonUnsafe.getUnsafe().getInt(baseAddress, baseOffset + offset); } return data; } @@ -230,7 +230,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage { long[] data = new long[getPageSize()]; for (int i = 0; i < data.length; i++) { long offset = i << longBits; - data[i] = CarbonUnsafe.unsafe.getLong(baseAddress, baseOffset + offset); + data[i] = CarbonUnsafe.getUnsafe().getLong(baseAddress, baseOffset + offset); } return data; } @@ -240,7 +240,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage { float[] data = new float[getPageSize()]; for (int i = 0; i < data.length; i++) { long offset = i << floatBits; - data[i] = CarbonUnsafe.unsafe.getFloat(baseAddress, baseOffset + offset); + data[i] = CarbonUnsafe.getUnsafe().getFloat(baseAddress, baseOffset + offset); } return data; } @@ -250,7 +250,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage { double[] data = new double[getPageSize()]; for (int i = 0; i < data.length; i++) { long offset = i << doubleBits; - data[i] = CarbonUnsafe.unsafe.getDouble(baseAddress, baseOffset + offset); + data[i] = CarbonUnsafe.getUnsafe().getDouble(baseAddress, baseOffset + offset); } return data; } @@ -267,43 +267,43 @@ public class UnsafeFixLengthColumnPage extends ColumnPage { @Override public void setBytePage(byte[] byteData) { - CarbonUnsafe.unsafe.copyMemory(byteData, CarbonUnsafe.BYTE_ARRAY_OFFSET, + CarbonUnsafe.getUnsafe().copyMemory(byteData, CarbonUnsafe.BYTE_ARRAY_OFFSET, baseAddress, baseOffset, byteData.length << byteBits); } @Override public void setShortPage(short[] shortData) { - CarbonUnsafe.unsafe.copyMemory(shortData, CarbonUnsafe.SHORT_ARRAY_OFFSET, + CarbonUnsafe.getUnsafe().copyMemory(shortData, CarbonUnsafe.SHORT_ARRAY_OFFSET, baseAddress, baseOffset, shortData.length << shortBits); } @Override public void setShortIntPage(byte[] shortIntData) { - CarbonUnsafe.unsafe.copyMemory(shortIntData, CarbonUnsafe.BYTE_ARRAY_OFFSET, + CarbonUnsafe.getUnsafe().copyMemory(shortIntData, CarbonUnsafe.BYTE_ARRAY_OFFSET, baseAddress, baseOffset, shortIntData.length); } @Override public void setIntPage(int[] intData) { - CarbonUnsafe.unsafe.copyMemory(intData, CarbonUnsafe.INT_ARRAY_OFFSET, + CarbonUnsafe.getUnsafe().copyMemory(intData, CarbonUnsafe.INT_ARRAY_OFFSET, baseAddress, baseOffset, intData.length << intBits); } @Override public void setLongPage(long[] longData) { - CarbonUnsafe.unsafe.copyMemory(longData, CarbonUnsafe.LONG_ARRAY_OFFSET, + CarbonUnsafe.getUnsafe().copyMemory(longData, CarbonUnsafe.LONG_ARRAY_OFFSET, baseAddress, baseOffset, longData.length << longBits); } @Override public void setFloatPage(float[] floatData) { - CarbonUnsafe.unsafe.copyMemory(floatData, CarbonUnsafe.FLOAT_ARRAY_OFFSET, + CarbonUnsafe.getUnsafe().copyMemory(floatData, CarbonUnsafe.FLOAT_ARRAY_OFFSET, baseAddress, baseOffset, floatData.length << floatBits); } @Override public void setDoublePage(double[] doubleData) { - CarbonUnsafe.unsafe.copyMemory(doubleData, CarbonUnsafe.DOUBLE_ARRAY_OFFSET, + CarbonUnsafe.getUnsafe().copyMemory(doubleData, CarbonUnsafe.DOUBLE_ARRAY_OFFSET, baseAddress, baseOffset, doubleData.length << doubleBits); } @@ -328,37 +328,37 @@ public class UnsafeFixLengthColumnPage extends ColumnPage { case BYTE: for (int i = 0; i < pageSize; i++) { long offset = i << byteBits; - codec.encode(i, CarbonUnsafe.unsafe.getByte(baseAddress, baseOffset + offset)); + codec.encode(i, CarbonUnsafe.getUnsafe().getByte(baseAddress, baseOffset + offset)); } break; case SHORT: for (int i = 0; i < pageSize; i++) { long offset = i << shortBits; - codec.encode(i, CarbonUnsafe.unsafe.getShort(baseAddress, baseOffset + offset)); + codec.encode(i, CarbonUnsafe.getUnsafe().getShort(baseAddress, baseOffset + offset)); } break; case INT: for (int i = 0; i < pageSize; i++) { long offset = i << intBits; - codec.encode(i, CarbonUnsafe.unsafe.getInt(baseAddress, baseOffset + offset)); + codec.encode(i, CarbonUnsafe.getUnsafe().getInt(baseAddress, baseOffset + offset)); } break; case LONG: for (int i = 0; i < pageSize; i++) { long offset = i << longBits; - codec.encode(i, CarbonUnsafe.unsafe.getLong(baseAddress, baseOffset + offset)); + codec.encode(i, CarbonUnsafe.getUnsafe().getLong(baseAddress, baseOffset + offset)); } break; case FLOAT: for (int i = 0; i < pageSize; i++) { long offset = i << floatBits; - codec.encode(i, CarbonUnsafe.unsafe.getFloat(baseAddress, baseOffset + offset)); + codec.encode(i, CarbonUnsafe.getUnsafe().getFloat(baseAddress, baseOffset + offset)); } break; case DOUBLE: for (int i = 0; i < pageSize; i++) { long offset = i << doubleBits; - codec.encode(i, CarbonUnsafe.unsafe.getDouble(baseAddress, baseOffset + offset)); + codec.encode(i, CarbonUnsafe.getUnsafe().getDouble(baseAddress, baseOffset + offset)); } break; default: @@ -366,8 +366,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage { } } - @Override - public byte[] compress(Compressor compressor) throws MemoryException, IOException { + @Override public byte[] compress(Compressor compressor) throws MemoryException, IOException { if (UnsafeMemoryManager.isOffHeap()) { // use raw compression and copy to byte[] int inputSize = pageSize * dataType.getSizeInBytes(); @@ -377,13 +376,13 @@ public class UnsafeFixLengthColumnPage extends ColumnPage { long outSize = compressor.rawCompress(baseOffset, inputSize, compressed.getBaseOffset()); assert outSize < Integer.MAX_VALUE; byte[] output = new byte[(int) outSize]; - CarbonUnsafe.unsafe.copyMemory(compressed.getBaseObject(), compressed.getBaseOffset(), output, - CarbonUnsafe.BYTE_ARRAY_OFFSET, outSize); + CarbonUnsafe.getUnsafe() + .copyMemory(compressed.getBaseObject(), compressed.getBaseOffset(), output, + CarbonUnsafe.BYTE_ARRAY_OFFSET, outSize); UnsafeMemoryManager.INSTANCE.freeMemory(taskId, compressed); return output; } else { return super.compress(compressor); } } - } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java index c89d29d..c7d21ef 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java @@ -95,7 +95,7 @@ public class UnsafeVarLengthColumnPage extends VarLengthColumnPageBase { if (totalLength + requestSize > capacity) { int newSize = 2 * capacity; MemoryBlock newBlock = UnsafeMemoryManager.allocateMemoryWithRetry(taskId, newSize); - CarbonUnsafe.unsafe.copyMemory(baseAddress, baseOffset, + CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset, newBlock.getBaseObject(), newBlock.getBaseOffset(), capacity); UnsafeMemoryManager.INSTANCE.freeMemory(taskId, memoryBlock); memoryBlock = newBlock; @@ -117,7 +117,7 @@ public class UnsafeVarLengthColumnPage extends VarLengthColumnPageBase { } catch (MemoryException e) { throw new RuntimeException(e); } - CarbonUnsafe.unsafe.copyMemory(bytes, CarbonUnsafe.BYTE_ARRAY_OFFSET + offset, + CarbonUnsafe.getUnsafe().copyMemory(bytes, CarbonUnsafe.BYTE_ARRAY_OFFSET + offset, baseAddress, baseOffset + rowOffset[rowId], length); } @@ -129,7 +129,7 @@ public class UnsafeVarLengthColumnPage extends VarLengthColumnPageBase { public BigDecimal getDecimal(int rowId) { int length = rowOffset[rowId + 1] - rowOffset[rowId]; byte[] bytes = new byte[length]; - CarbonUnsafe.unsafe.copyMemory(baseAddress, baseOffset + rowOffset[rowId], + CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset[rowId], bytes, CarbonUnsafe.BYTE_ARRAY_OFFSET, length); return decimalConverter.getDecimal(bytes); @@ -141,7 +141,7 @@ public class UnsafeVarLengthColumnPage extends VarLengthColumnPageBase { for (int rowId = 0; rowId < pageSize; rowId++) { int length = rowOffset[rowId + 1] - rowOffset[rowId]; byte[] rowData = new byte[length]; - CarbonUnsafe.unsafe.copyMemory(baseAddress, baseOffset + rowOffset[rowId], + CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset[rowId], rowData, CarbonUnsafe.BYTE_ARRAY_OFFSET, length); bytes[rowId] = rowData; } @@ -150,7 +150,7 @@ public class UnsafeVarLengthColumnPage extends VarLengthColumnPageBase { @Override void copyBytes(int rowId, byte[] dest, int destOffset, int length) { - CarbonUnsafe.unsafe.copyMemory(baseAddress, baseOffset + rowOffset[rowId], + CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset[rowId], dest, CarbonUnsafe.BYTE_ARRAY_OFFSET + destOffset, length); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DeltaIntegralCodec.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DeltaIntegralCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DeltaIntegralCodec.java index ba61cd3..80545b9 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DeltaIntegralCodec.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DeltaIntegralCodec.java @@ -44,8 +44,8 @@ public class DeltaIntegralCodec extends AdaptiveCompressionCodec { return new DeltaIntegralCodec(srcDataType, targetDataType, stats, compressor); } - private DeltaIntegralCodec(DataType srcDataType, DataType targetDataType, - SimpleStatsResult stats, Compressor compressor) { + private DeltaIntegralCodec(DataType srcDataType, DataType targetDataType, SimpleStatsResult stats, + Compressor compressor) { super(srcDataType, targetDataType, stats, compressor); switch (srcDataType) { case BYTE: @@ -62,8 +62,11 @@ public class DeltaIntegralCodec extends AdaptiveCompressionCodec { break; case FLOAT: case DOUBLE: - max = (long)((double) stats.getMax()); + max = (long) ((double) stats.getMax()); break; + default: + throw new UnsupportedOperationException( + "unsupported data type for Delta compress: " + srcDataType); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/RLECodec.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/RLECodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/RLECodec.java index 46c5c58..dda89e0 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/RLECodec.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/RLECodec.java @@ -326,6 +326,8 @@ public class RLECodec implements ColumnPageCodec { case LONG: decodeLongPage(in, resultPage); break; + default: + throw new RuntimeException("unsupported datatype:" + dataType); } return resultPage; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java index 918039a..927ab5f 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java @@ -88,6 +88,9 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector, Si instance.scale = meta.getScale(); instance.precision = meta.getPrecision(); break; + default: + throw new UnsupportedOperationException( + "unsupported data type for stats collection: " + meta.getSrcDataType()); } return instance; } @@ -125,6 +128,9 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector, Si instance.scale = -1; instance.precision = -1; break; + default: + throw new UnsupportedOperationException( + "unsupported data type for Stats collection: " + meta.getType()); } return instance; } @@ -159,6 +165,10 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector, Si decimal = scale; this.scale = scale; this.precision = precision; + break; + default: + throw new UnsupportedOperationException( + "unsupported data type for Stats collection: " + dataType); } } @@ -192,6 +202,9 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector, Si minDecimal = (minDecimal.compareTo(zeroDecimal) < 0) ? minDecimal : zeroDecimal; } break; + default: + throw new UnsupportedOperationException( + "unsupported data type for Stats collection: " + dataType); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/row/LoadStatusType.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/row/LoadStatusType.java b/core/src/main/java/org/apache/carbondata/core/datastore/row/LoadStatusType.java index cbdeadb..d3c5a94 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/row/LoadStatusType.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/row/LoadStatusType.java @@ -34,8 +34,4 @@ public enum LoadStatusType { public String getMessage() { return message; } - - public void setMessage(String message) { - this.message = message; - } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClientHandler.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClientHandler.java b/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClientHandler.java index 9922523..01ef59a 100644 --- a/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClientHandler.java +++ b/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClientHandler.java @@ -59,7 +59,7 @@ public class DictionaryClientHandler extends ChannelInboundHandlerAdapter { DictionaryMessage key = new DictionaryMessage(); key.readData(data); data.release(); - responseMsgQueue.offer(key); + responseMsgQueue.add(key); } catch (Exception e) { LOGGER.error(e); throw e; http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGenerator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGenerator.java b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGenerator.java index 9246830..221f9b7 100644 --- a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGenerator.java +++ b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGenerator.java @@ -16,6 +16,7 @@ */ package org.apache.carbondata.core.dictionary.generator; +import java.util.Iterator; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -71,8 +72,14 @@ public class ServerDictionaryGenerator implements DictionaryGenerator<Integer, D } public void writeDictionaryData() throws Exception { - for (String tableUniqueName: tableMap.keySet()) { - TableDictionaryGenerator generator = tableMap.get(tableUniqueName); + final Iterator<Map.Entry<String, TableDictionaryGenerator>> iterator = + tableMap.entrySet().iterator(); + String tableUniqueName; + TableDictionaryGenerator generator; + while (iterator.hasNext()) { + Map.Entry<String, TableDictionaryGenerator> entry = iterator.next(); + tableUniqueName = entry.getKey(); + generator = entry.getValue(); generator.writeDictionaryData(tableUniqueName); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGenerator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGenerator.java b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGenerator.java index 56ed7b9..ee3a2fe 100644 --- a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGenerator.java +++ b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGenerator.java @@ -16,14 +16,11 @@ */ package org.apache.carbondata.core.dictionary.generator; -import java.util.ArrayList; -import java.util.List; +import java.io.IOException; import java.util.Map; -import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import org.apache.carbondata.common.logging.LogService; @@ -83,7 +80,6 @@ public class TableDictionaryGenerator @Override public void writeDictionaryData(String tableUniqueName) { int numOfCores = 1; - final String tableName = tableUniqueName; try { numOfCores = Integer.parseInt(CarbonProperties.getInstance() .getProperty(CarbonCommonConstants.NUM_CORES_LOADING, @@ -92,16 +88,9 @@ public class TableDictionaryGenerator numOfCores = Integer.parseInt(CarbonCommonConstants.NUM_CORES_DEFAULT_VAL); } long start = System.currentTimeMillis(); - List<Future<Void>> taskSubmitList = - new ArrayList<>(columnMap.size()); ExecutorService executorService = Executors.newFixedThreadPool(numOfCores); - for (final DictionaryGenerator generator: columnMap.values()) { - taskSubmitList.add(executorService.submit(new Callable<Void>() { - @Override public Void call() throws Exception { - ((DictionaryWriter) (generator)).writeDictionaryData(tableName); - return null; - } - })); + for (final DictionaryGenerator generator : columnMap.values()) { + executorService.execute(new WriteDictionaryDataRunnable(generator, tableUniqueName)); } try { @@ -125,4 +114,22 @@ public class TableDictionaryGenerator } } } + + private static class WriteDictionaryDataRunnable implements Runnable { + private final DictionaryGenerator generator; + private final String tableUniqueName; + + public WriteDictionaryDataRunnable(DictionaryGenerator generator, String tableUniqueName) { + this.generator = generator; + this.tableUniqueName = tableUniqueName; + } + + @Override public void run() { + try { + ((DictionaryWriter)generator).writeDictionaryData(tableUniqueName); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/dictionary/server/DictionaryServer.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/server/DictionaryServer.java b/core/src/main/java/org/apache/carbondata/core/dictionary/server/DictionaryServer.java index 84f2a0d..617038e 100644 --- a/core/src/main/java/org/apache/carbondata/core/dictionary/server/DictionaryServer.java +++ b/core/src/main/java/org/apache/carbondata/core/dictionary/server/DictionaryServer.java @@ -47,7 +47,7 @@ public class DictionaryServer { private EventLoopGroup worker; private int port; private static Object lock = new Object(); - private static DictionaryServer INSTANCE = null; + private static volatile DictionaryServer INSTANCE = null; private DictionaryServer(int port) { startServer(port); http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java b/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java index 66da4d0..c3a72f0 100644 --- a/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java +++ b/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java @@ -31,7 +31,7 @@ import org.apache.hadoop.fs.RemoteIterator; */ public class Blocklet implements Serializable { - private Path path; + private String path; private String segmentId; @@ -44,12 +44,12 @@ public class Blocklet implements Serializable { private String[] location; public Blocklet(String path, String blockletId) { - this.path = new Path(path); + this.path = path; this.blockletId = blockletId; } public Path getPath() { - return path; + return new Path(path); } public String getBlockletId() { @@ -65,8 +65,9 @@ public class Blocklet implements Serializable { } public void updateLocations() throws IOException { - FileSystem fs = path.getFileSystem(FileFactory.getConfiguration()); - RemoteIterator<LocatedFileStatus> iter = fs.listLocatedStatus(path); + Path fspath = new Path(path); + FileSystem fs = fspath.getFileSystem(FileFactory.getConfiguration()); + RemoteIterator<LocatedFileStatus> iter = fs.listLocatedStatus(fspath); LocatedFileStatus fileStatus = iter.next(); location = fileStatus.getBlockLocations()[0].getHosts(); length = fileStatus.getLen(); http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/indexstore/UnsafeMemoryDMStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/UnsafeMemoryDMStore.java b/core/src/main/java/org/apache/carbondata/core/indexstore/UnsafeMemoryDMStore.java index eb67294..5d17426 100644 --- a/core/src/main/java/org/apache/carbondata/core/indexstore/UnsafeMemoryDMStore.java +++ b/core/src/main/java/org/apache/carbondata/core/indexstore/UnsafeMemoryDMStore.java @@ -25,7 +25,7 @@ import org.apache.carbondata.core.memory.UnsafeMemoryManager; import org.apache.carbondata.core.util.ThreadLocalTaskInfo; import static org.apache.carbondata.core.memory.CarbonUnsafe.BYTE_ARRAY_OFFSET; -import static org.apache.carbondata.core.memory.CarbonUnsafe.unsafe; +import static org.apache.carbondata.core.memory.CarbonUnsafe.getUnsafe; /** * Store the data map row @{@link DataMapRow} data to unsafe. @@ -67,7 +67,7 @@ public class UnsafeMemoryDMStore { if (runningLength + rowSize >= allocatedSize) { MemoryBlock allocate = UnsafeMemoryManager.allocateMemoryWithRetry(taskId, allocatedSize + capacity); - unsafe.copyMemory(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset(), + getUnsafe().copyMemory(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset(), allocate.getBaseObject(), allocate.getBaseOffset(), runningLength); UnsafeMemoryManager.INSTANCE.freeMemory(taskId, memoryBlock); allocatedSize = allocatedSize + capacity; @@ -104,51 +104,57 @@ public class UnsafeMemoryDMStore { case FIXED: switch (schema.getDataType()) { case BYTE: - unsafe.putByte(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength, - row.getByte(index)); + getUnsafe() + .putByte(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength, + row.getByte(index)); runningLength += row.getSizeInBytes(index); break; case SHORT: - unsafe + getUnsafe() .putShort(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength, row.getShort(index)); runningLength += row.getSizeInBytes(index); break; case INT: - unsafe.putInt(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength, - row.getInt(index)); + getUnsafe() + .putInt(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength, + row.getInt(index)); runningLength += row.getSizeInBytes(index); break; case LONG: - unsafe.putLong(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength, - row.getLong(index)); + getUnsafe() + .putLong(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength, + row.getLong(index)); runningLength += row.getSizeInBytes(index); break; case FLOAT: - unsafe + getUnsafe() .putFloat(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength, row.getFloat(index)); runningLength += row.getSizeInBytes(index); break; case DOUBLE: - unsafe + getUnsafe() .putDouble(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength, row.getDouble(index)); runningLength += row.getSizeInBytes(index); break; case BYTE_ARRAY: byte[] data = row.getByteArray(index); - unsafe.copyMemory(data, BYTE_ARRAY_OFFSET, memoryBlock.getBaseObject(), + getUnsafe().copyMemory(data, BYTE_ARRAY_OFFSET, memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength, data.length); runningLength += row.getSizeInBytes(index); break; + default: + throw new UnsupportedOperationException( + "unsupported data type for unsafe storage: " + schema.getDataType()); } break; case VARIABLE: byte[] data = row.getByteArray(index); - unsafe.putShort(memoryBlock.getBaseOffset() + runningLength, (short) data.length); + getUnsafe().putShort(memoryBlock.getBaseOffset() + runningLength, (short) data.length); runningLength += 2; - unsafe.copyMemory(data, BYTE_ARRAY_OFFSET, memoryBlock.getBaseObject(), + getUnsafe().copyMemory(data, BYTE_ARRAY_OFFSET, memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength, data.length); runningLength += data.length; break; @@ -160,6 +166,9 @@ public class UnsafeMemoryDMStore { addToUnsafe(childSchemas[i], struct, i); } break; + default: + throw new UnsupportedOperationException( + "unsupported data type for unsafe storage: " + schema.getDataType()); } } @@ -172,7 +181,7 @@ public class UnsafeMemoryDMStore { if (runningLength < allocatedSize) { MemoryBlock allocate = UnsafeMemoryManager.allocateMemoryWithRetry(taskId, runningLength); - unsafe.copyMemory(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset(), + getUnsafe().copyMemory(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset(), allocate.getBaseObject(), allocate.getBaseOffset(), runningLength); UnsafeMemoryManager.INSTANCE.freeMemory(taskId, memoryBlock); memoryBlock = allocate; http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java index 2e82c46..928c78b 100644 --- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java +++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java @@ -24,6 +24,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Arrays; import java.util.BitSet; import java.util.Comparator; import java.util.List; @@ -31,6 +32,7 @@ import java.util.List; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.cache.Cacheable; +import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.datamap.DataMapDistributable; import org.apache.carbondata.core.datamap.dev.DataMap; import org.apache.carbondata.core.datastore.IndexKey; @@ -131,7 +133,8 @@ public class BlockletDataMap implements DataMap, Cacheable { row.setInt(blockletInfo.getNumberOfRows(), ordinal++); // add file path - byte[] filePathBytes = filePath.getBytes(); + byte[] filePathBytes = + filePath.getBytes(CarbonCommonConstants.DEFAULT_CHARSET_CLASS); row.setByteArray(filePathBytes, ordinal++); // add pages @@ -244,10 +247,11 @@ public class BlockletDataMap implements DataMap, Cacheable { } if (LOGGER.isDebugEnabled()) { LOGGER.debug( - "Successfully retrieved the start and end key" + "Dictionary Start Key: " + searchStartKey - .getDictionaryKeys() + "No Dictionary Start Key " + searchStartKey - .getNoDictionaryKeys() + "Dictionary End Key: " + searchEndKey.getDictionaryKeys() - + "No Dictionary End Key " + searchEndKey.getNoDictionaryKeys()); + "Successfully retrieved the start and end key" + "Dictionary Start Key: " + Arrays + .toString(searchStartKey.getDictionaryKeys()) + "No Dictionary Start Key " + Arrays + .toString(searchStartKey.getNoDictionaryKeys()) + "Dictionary End Key: " + Arrays + .toString(searchEndKey.getDictionaryKeys()) + "No Dictionary End Key " + Arrays + .toString(searchEndKey.getNoDictionaryKeys())); } if (filterExp == null) { int rowCount = unsafeMemoryDMStore.getRowCount(); @@ -284,8 +288,9 @@ public class BlockletDataMap implements DataMap, Cacheable { } private Blocklet createBlocklet(DataMapRow row, int blockletId) { - Blocklet blocklet = - new Blocklet(new String(row.getByteArray(FILE_PATH_INDEX)), blockletId + ""); + Blocklet blocklet = new Blocklet( + new String(row.getByteArray(FILE_PATH_INDEX), CarbonCommonConstants.DEFAULT_CHARSET_CLASS), + blockletId + ""); BlockletDetailInfo detailInfo = new BlockletDetailInfo(); detailInfo.setRowCount(row.getInt(ROW_COUNT_INDEX)); detailInfo.setPagesCount(row.getShort(PAGE_COUNT_INDEX)); http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java index 5509c75..8896e5d 100644 --- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java +++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java @@ -52,7 +52,7 @@ public class BlockletDataRefNodeWrapper implements DataRefNode { @Override public DataRefNode getNextDataRefNode() { if (index + 1 < blockInfos.size()) { - new BlockletDataRefNodeWrapper(blockInfos, index + 1, dimensionLens); + return new BlockletDataRefNodeWrapper(blockInfos, index + 1, dimensionLens); } return null; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/indexstore/row/UnsafeDataMapRow.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/row/UnsafeDataMapRow.java b/core/src/main/java/org/apache/carbondata/core/indexstore/row/UnsafeDataMapRow.java index c398115..2c76990 100644 --- a/core/src/main/java/org/apache/carbondata/core/indexstore/row/UnsafeDataMapRow.java +++ b/core/src/main/java/org/apache/carbondata/core/indexstore/row/UnsafeDataMapRow.java @@ -21,7 +21,7 @@ import org.apache.carbondata.core.indexstore.schema.DataMapSchema; import org.apache.carbondata.core.memory.MemoryBlock; import static org.apache.carbondata.core.memory.CarbonUnsafe.BYTE_ARRAY_OFFSET; -import static org.apache.carbondata.core.memory.CarbonUnsafe.unsafe; +import static org.apache.carbondata.core.memory.CarbonUnsafe.getUnsafe; /** * Unsafe implementation of data map row. @@ -43,14 +43,15 @@ public class UnsafeDataMapRow extends DataMapRow { int position = getPosition(ordinal); switch (schemas[ordinal].getSchemaType()) { case VARIABLE: - length = unsafe.getShort(block.getBaseObject(), block.getBaseOffset() + pointer + position); + length = + getUnsafe().getShort(block.getBaseObject(), block.getBaseOffset() + pointer + position); position += 2; break; default: length = schemas[ordinal].getLength(); } byte[] data = new byte[length]; - unsafe.copyMemory(block.getBaseObject(), block.getBaseOffset() + pointer + position, data, + getUnsafe().copyMemory(block.getBaseObject(), block.getBaseOffset() + pointer + position, data, BYTE_ARRAY_OFFSET, data.length); return data; } @@ -60,7 +61,8 @@ public class UnsafeDataMapRow extends DataMapRow { int position = getPosition(ordinal); switch (schemas[ordinal].getSchemaType()) { case VARIABLE: - length = unsafe.getShort(block.getBaseObject(), block.getBaseOffset() + pointer + position); + length = + getUnsafe().getShort(block.getBaseObject(), block.getBaseOffset() + pointer + position); break; default: length = schemas[ordinal].getLength(); @@ -72,7 +74,8 @@ public class UnsafeDataMapRow extends DataMapRow { int length; switch (schemas[ordinal].getSchemaType()) { case VARIABLE: - length = unsafe.getShort(block.getBaseObject(), block.getBaseOffset() + pointer + position); + length = + getUnsafe().getShort(block.getBaseObject(), block.getBaseOffset() + pointer + position); break; default: length = schemas[ordinal].getLength(); @@ -91,7 +94,7 @@ public class UnsafeDataMapRow extends DataMapRow { } @Override public int getInt(int ordinal) { - return unsafe + return getUnsafe() .getInt(block.getBaseObject(), block.getBaseOffset() + pointer + getPosition(ordinal)); } @@ -104,7 +107,7 @@ public class UnsafeDataMapRow extends DataMapRow { } @Override public byte getByte(int ordinal) { - return unsafe + return getUnsafe() .getByte(block.getBaseObject(), block.getBaseOffset() + pointer + getPosition(ordinal)); } @@ -113,7 +116,7 @@ public class UnsafeDataMapRow extends DataMapRow { } @Override public short getShort(int ordinal) { - return unsafe + return getUnsafe() .getShort(block.getBaseObject(), block.getBaseOffset() + pointer + getPosition(ordinal)); } @@ -122,7 +125,7 @@ public class UnsafeDataMapRow extends DataMapRow { } @Override public long getLong(int ordinal) { - return unsafe + return getUnsafe() .getLong(block.getBaseObject(), block.getBaseOffset() + pointer + getPosition(ordinal)); } @@ -131,7 +134,7 @@ public class UnsafeDataMapRow extends DataMapRow { } @Override public float getFloat(int ordinal) { - return unsafe + return getUnsafe() .getFloat(block.getBaseObject(), block.getBaseOffset() + pointer + getPosition(ordinal)); } @@ -140,7 +143,7 @@ public class UnsafeDataMapRow extends DataMapRow { } @Override public double getDouble(int ordinal) { - return unsafe + return getUnsafe() .getDouble(block.getBaseObject(), block.getBaseOffset() + pointer + getPosition(ordinal)); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/keygenerator/columnar/impl/MultiDimKeyVarLengthEquiSplitGenerator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/columnar/impl/MultiDimKeyVarLengthEquiSplitGenerator.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/columnar/impl/MultiDimKeyVarLengthEquiSplitGenerator.java index f628bca..72b250b 100644 --- a/core/src/main/java/org/apache/carbondata/core/keygenerator/columnar/impl/MultiDimKeyVarLengthEquiSplitGenerator.java +++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/columnar/impl/MultiDimKeyVarLengthEquiSplitGenerator.java @@ -109,29 +109,6 @@ public class MultiDimKeyVarLengthEquiSplitGenerator extends MultiDimKeyVarLength dimBlockArray[j] = convertToArray(dimBlockSet[j]); } - int[][] splitDimArrayLocalIndexes = new int[splitDimArray.length][]; - for (int j = 0; j < splitDimArrayLocalIndexes.length; j++) { - splitDimArrayLocalIndexes[j] = splitDimArray[j].length > 0 ? - new int[] { 0, splitDimArray[j][1] - splitDimArray[j][0] } : - new int[0]; - } - - int[][][] byteRangesForDims = new int[byteRangesForKeys.length][][]; - for (int j = 0; j < byteRangesForKeys.length; j++) { - if (dimBlockArray[j].length > 1) { - int[] bArray1 = splitDimArrayLocalIndexes[dimBlockArray[j][0]]; - byteRangesForDims[j] = new int[2][2]; - byteRangesForDims[j][0] = - new int[] { bArray1[bArray1.length - 1], bArray1[bArray1.length - 1] }; - byteRangesForDims[j][1] = new int[] { 0, - (byteRangesForKeys[j][byteRangesForKeys[j].length - 1] - byteRangesForKeys[j][0]) - 1 }; - } else { - byteRangesForDims[j] = new int[1][1]; - int[] bArray1 = splitDimArray[dimBlockArray[j][0]]; - byteRangesForDims[j][0] = new int[] { byteRangesForKeys[j][0] - bArray1[0], - byteRangesForKeys[j][1] - bArray1[0] }; - } - } blockKeySize = new int[splitDimArray.length]; for (int j = 0; j < blockKeySize.length; j++) {
