[CARBONDATA-1326] Fixed high priority findbug issues Fixed high priority findbug issues in the code
This closes #1191 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/2b66476d Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/2b66476d Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/2b66476d Branch: refs/heads/master Commit: 2b66476dd643bad15aa4713b84bc84afe2233c35 Parents: aadbd5c Author: manishgupta88 <[email protected]> Authored: Fri Jul 21 21:48:20 2017 +0800 Committer: Raghunandan S <[email protected]> Committed: Tue Jul 25 17:38:57 2017 +0800 ---------------------------------------------------------------------- .../cache/dictionary/ColumnDictionaryInfo.java | 10 +++++--- .../dictionary/DoubleArrayTrieDictionary.java | 7 ++++-- .../core/constants/CarbonCommonConstants.java | 13 +++++----- .../core/datastore/block/BlockInfo.java | 4 ++- .../block/SegmentTaskIndexWrapper.java | 17 ------------- .../core/datastore/block/TableBlockInfo.java | 7 ++++-- .../core/datastore/page/ColumnPage.java | 2 +- .../generator/key/DictionaryMessage.java | 16 +++++++----- .../core/indexstore/DataMapStoreManager.java | 2 +- .../datatype/DecimalConverterFactory.java | 2 +- .../core/mutate/DeleteDeltaBlockletDetails.java | 6 +++-- .../scan/filter/FilterExpressionProcessor.java | 26 +++++++++++++++++--- .../scan/filter/intf/FilterOptimizerImpl.java | 2 +- .../RowLevelRangeFilterResolverImpl.java | 2 +- .../core/scan/result/AbstractScannedResult.java | 7 ++++-- .../statusmanager/SegmentStatusManager.java | 5 +++- .../SegmentUpdateStatusManager.java | 3 --- .../CarbonDictionarySortInfoPreparator.java | 4 +-- .../carbondata/hadoop/CarbonInputSplit.java | 7 ++++-- .../internal/index/impl/InMemoryBTreeIndex.java | 2 +- .../carbondata/spark/util/CarbonScalaUtil.scala | 4 ++- .../org/apache/spark/util/TableAPIUtil.scala | 2 +- .../processing/csvload/CSVInputFormat.java | 5 +++- .../newflow/AbstractDataLoadProcessorStep.java | 26 ++++++++++---------- .../CarbonRowDataWriterProcessorStepImpl.java | 1 + .../steps/DataConverterProcessorStepImpl.java | 1 + ...ConverterProcessorWithBucketingStepImpl.java | 1 + .../steps/DataWriterBatchProcessorStepImpl.java | 1 + .../steps/DataWriterProcessorStepImpl.java | 1 + .../newflow/steps/InputProcessorStepImpl.java | 5 ++-- .../newflow/steps/SortProcessorStepImpl.java | 1 + .../writer/v2/CarbonFactDataWriterImplV2.java | 4 ++- 32 files changed, 117 insertions(+), 79 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ColumnDictionaryInfo.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ColumnDictionaryInfo.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ColumnDictionaryInfo.java index d3599da..260ba90 100644 --- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ColumnDictionaryInfo.java +++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ColumnDictionaryInfo.java @@ -194,8 +194,9 @@ public class ColumnDictionaryInfo extends AbstractColumnDictionaryInfo { byte[] dictionaryValue = getDictionaryBytesFromSurrogate(surrogateKey); int cmp = -1; if (this.getDataType() != DataType.STRING) { - cmp = compareFilterKeyWithDictionaryKey(new String(dictionaryValue), filterKey, - this.getDataType()); + cmp = compareFilterKeyWithDictionaryKey( + new String(dictionaryValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)), + filterKey, this.getDataType()); } else { cmp = ByteUtil.UnsafeComparer.INSTANCE.compareTo(dictionaryValue, key); @@ -240,8 +241,9 @@ public class ColumnDictionaryInfo extends AbstractColumnDictionaryInfo { if (null == dictionaryValue) { cmp = -1; } else if (this.getDataType() != DataType.STRING) { - cmp = compareFilterKeyWithDictionaryKey(new String(dictionaryValue), filterKey, - this.getDataType()); + cmp = compareFilterKeyWithDictionaryKey( + new String(dictionaryValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)), + filterKey, this.getDataType()); } else { cmp = http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java index f738008..cab1926 100644 --- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java +++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java @@ -21,9 +21,12 @@ import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.IOException; import java.io.PrintStream; +import java.nio.charset.Charset; import java.util.SortedSet; import java.util.TreeSet; +import org.apache.carbondata.core.constants.CarbonCommonConstants; + /** * A dictionary based on DoubleArrayTrie data structure that maps enumerations * of byte[] to int IDs. With DoubleArrayTrie the memory footprint of the mapping @@ -116,7 +119,7 @@ public class DoubleArrayTrieDictionary { */ public int getValue(String key) { String k = key + '\0'; - byte[] bKeys = k.getBytes(); + byte[] bKeys = k.getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)); return getValue(bKeys); } @@ -315,7 +318,7 @@ public class DoubleArrayTrieDictionary { */ public boolean insert(String key) { String k = key + '\0'; - byte[] bKeys = k.getBytes(); + byte[] bKeys = k.getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)); if (!insert(bKeys)) { return false; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java index 8c61305..8427a22 100644 --- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java +++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java @@ -176,9 +176,15 @@ public final class CarbonCommonConstants { public static final String MEMBER_DEFAULT_VAL = "@NU#LL$!"; /** + * default charset to be used for reading and writing + */ + public static final String DEFAULT_CHARSET = "UTF-8"; + + /** * MEMBER_DEFAULT_VAL_ARRAY */ - public static final byte[] MEMBER_DEFAULT_VAL_ARRAY = MEMBER_DEFAULT_VAL.getBytes(); + public static final byte[] MEMBER_DEFAULT_VAL_ARRAY = + MEMBER_DEFAULT_VAL.getBytes(Charset.forName(DEFAULT_CHARSET)); /** * Bytes for string 0, it is used in codegen in case of null values. @@ -772,11 +778,6 @@ public final class CarbonCommonConstants { public static final String ENABLE_XXHASH_DEFAULT = "true"; /** - * default charset to be used for reading and writing - */ - public static final String DEFAULT_CHARSET = "UTF-8"; - - /** * default charset class to be used for reading and writing */ public static final Charset DEFAULT_CHARSET_CLASS = Charset.forName(DEFAULT_CHARSET); http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/core/src/main/java/org/apache/carbondata/core/datastore/block/BlockInfo.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/BlockInfo.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/BlockInfo.java index 2833316..d2eae8b 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/block/BlockInfo.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/BlockInfo.java @@ -16,6 +16,8 @@ */ package org.apache.carbondata.core.datastore.block; +import java.io.Serializable; + import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.util.path.CarbonTablePath; @@ -26,7 +28,7 @@ import org.apache.carbondata.core.util.path.CarbonTablePath; * block will be loaded together. This class will be used to store table block info * and equals and hash code method is used to identify blocklet belongs to same block */ -public class BlockInfo { +public class BlockInfo implements Serializable { /** * table block info, stores all the details http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexWrapper.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexWrapper.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexWrapper.java index 516cd6a..4fe6d1b 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexWrapper.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexWrapper.java @@ -23,7 +23,6 @@ import java.util.concurrent.atomic.AtomicLong; import org.apache.carbondata.core.cache.Cacheable; import org.apache.carbondata.core.datastore.SegmentTaskIndexStore; -import org.apache.carbondata.core.mutate.UpdateVO; /** * SegmentTaskIndexWrapper class holds the taskIdToTableSegmentMap @@ -45,7 +44,6 @@ public class SegmentTaskIndexWrapper implements Cacheable { protected AtomicLong memorySize = new AtomicLong(); private Long refreshedTimeStamp; - private UpdateVO invalidTaskKey; public SegmentTaskIndexWrapper( Map<SegmentTaskIndexStore.TaskBucketHolder, AbstractIndex> taskIdToTableSegmentMap) { this.taskIdToTableSegmentMap = taskIdToTableSegmentMap; @@ -128,19 +126,4 @@ public class SegmentTaskIndexWrapper implements Cacheable { this.refreshedTimeStamp = refreshedTimeStamp; } - public void removeEntryFromCacheAndRefresh(String taskId) { - AbstractIndex blockEntry = this.getTaskIdToTableSegmentMap().remove(taskId); - if (null != blockEntry) { - memorySize.set(memorySize.get() - blockEntry.getMemorySize()); - } - } - - public void setLastUpdateVO(UpdateVO invalidTaskKey) { - this.invalidTaskKey = invalidTaskKey; - } - - public UpdateVO getInvalidTaskKey() { - return invalidTaskKey; - } - } http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java index 44347cf..1da6699 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java @@ -17,6 +17,7 @@ package org.apache.carbondata.core.datastore.block; import java.io.Serializable; +import java.nio.charset.Charset; import java.util.HashMap; import java.util.Map; @@ -221,8 +222,10 @@ public class TableBlockInfo implements Distributable, Serializable { // offset of // the file if (CarbonTablePath.isCarbonDataFile(filePath)) { - int compare = ByteUtil.compare(DataFileUtil.getTaskNo(filePath).getBytes(), - DataFileUtil.getTaskNo(((TableBlockInfo) other).filePath).getBytes()); + int compare = ByteUtil.compare(DataFileUtil.getTaskNo(filePath) + .getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)), + DataFileUtil.getTaskNo(((TableBlockInfo) other).filePath) + .getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET))); if (compare != 0) { return compare; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java index 2c43165..ef2c9ec 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java @@ -109,7 +109,7 @@ public abstract class ColumnPage { } private static ColumnPage createPage(DataType dataType, int pageSize, int scale, int precision) { - if (dataType.equals(BYTE_ARRAY) | dataType.equals(DECIMAL)) { + if (dataType.equals(BYTE_ARRAY) || dataType.equals(DECIMAL)) { return createVarLengthPage(dataType, pageSize, scale, precision); } else { return createFixLengthPage(dataType, pageSize, scale, precision); http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/core/src/main/java/org/apache/carbondata/core/dictionary/generator/key/DictionaryMessage.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/key/DictionaryMessage.java b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/key/DictionaryMessage.java index 2ae0521..0bb82f9 100644 --- a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/key/DictionaryMessage.java +++ b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/key/DictionaryMessage.java @@ -16,6 +16,8 @@ */ package org.apache.carbondata.core.dictionary.generator.key; +import java.nio.charset.Charset; + import org.apache.carbondata.core.constants.CarbonCommonConstants; import io.netty.buffer.ByteBuf; @@ -53,11 +55,12 @@ public class DictionaryMessage { public void readData(ByteBuf byteBuf) { byte[] tableBytes = new byte[byteBuf.readInt()]; byteBuf.readBytes(tableBytes); - tableUniqueName = new String(tableBytes); + tableUniqueName = + new String(tableBytes, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)); byte[] colBytes = new byte[byteBuf.readInt()]; byteBuf.readBytes(colBytes); - columnName = new String(colBytes); + columnName = new String(colBytes, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)); byte typeByte = byteBuf.readByte(); type = getKeyType(typeByte); @@ -68,7 +71,7 @@ public class DictionaryMessage { } else { byte[] dataBytes = new byte[byteBuf.readInt()]; byteBuf.readBytes(dataBytes); - data = new String(dataBytes); + data = new String(dataBytes, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)); } } @@ -77,11 +80,12 @@ public class DictionaryMessage { // Just reserve the bytes to add length of header at last. byteBuf.writeShort(Short.MAX_VALUE); - byte[] tableBytes = tableUniqueName.getBytes(); + byte[] tableBytes = + tableUniqueName.getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)); byteBuf.writeInt(tableBytes.length); byteBuf.writeBytes(tableBytes); - byte[] colBytes = columnName.getBytes(); + byte[] colBytes = columnName.getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)); byteBuf.writeInt(colBytes.length); byteBuf.writeBytes(colBytes); @@ -92,7 +96,7 @@ public class DictionaryMessage { byteBuf.writeInt(dictionaryValue); } else { byteBuf.writeByte(1); - byte[] dataBytes = data.getBytes(); + byte[] dataBytes = data.getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)); byteBuf.writeInt(dataBytes.length); byteBuf.writeBytes(dataBytes); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapStoreManager.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapStoreManager.java b/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapStoreManager.java index 06638ad..64c6e20 100644 --- a/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapStoreManager.java +++ b/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapStoreManager.java @@ -86,7 +86,7 @@ public class DataMapStoreManager { } catch (Exception e) { LOGGER.error(e); } - dataMap.init(identifier, dataMapName); + // TODO: Initialize a data map by calling init method on the data map map.put(dataMapName, dataMap); return dataMap; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java index 555df1c..459eb24 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java @@ -29,7 +29,7 @@ import org.apache.carbondata.core.util.DataTypeUtil; */ public final class DecimalConverterFactory { - public static DecimalConverterFactory INSTANCE = new DecimalConverterFactory(); + public static final DecimalConverterFactory INSTANCE = new DecimalConverterFactory(); private int[] minBytesForPrecision = minBytesForPrecision(); http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockletDetails.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockletDetails.java b/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockletDetails.java index 0f54f3a..22785fa 100644 --- a/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockletDetails.java +++ b/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockletDetails.java @@ -69,9 +69,11 @@ public class DeleteDeltaBlockletDetails implements Serializable { if (obj == null || !(obj instanceof DeleteDeltaBlockletDetails)) { return false; } - DeleteDeltaBlockletDetails that = (DeleteDeltaBlockletDetails) obj; - return id.equals(that.id) && pageId == that.pageId; + if (null == pageId || null == that.pageId) { + return false; + } + return id.equals(that.id) && (pageId.intValue() == that.pageId.intValue()); } @Override public int hashCode() { http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java index 21c7bf6..02052aa 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java @@ -131,11 +131,14 @@ public class FilterExpressionProcessor implements FilterProcessor { } } if (LOGGER.isDebugEnabled()) { + char delimiter = ','; LOGGER.debug( - "Successfully retrieved the start and end key" + "Dictionary Start Key: " + searchStartKey - .getDictionaryKeys() + "No Dictionary Start Key " + searchStartKey - .getNoDictionaryKeys() + "Dictionary End Key: " + searchEndKey.getDictionaryKeys() - + "No Dictionary End Key " + searchEndKey.getNoDictionaryKeys()); + "Successfully retrieved the start and end key" + "Dictionary Start Key: " + joinByteArray( + searchStartKey.getDictionaryKeys(), delimiter) + "No Dictionary Start Key " + + joinByteArray(searchStartKey.getNoDictionaryKeys(), delimiter) + + "Dictionary End Key: " + joinByteArray(searchEndKey.getDictionaryKeys(), delimiter) + + "No Dictionary End Key " + joinByteArray(searchEndKey.getNoDictionaryKeys(), + delimiter)); } long startTimeInMillis = System.currentTimeMillis(); DataRefNodeFinder blockFinder = new BTreeDataRefNodeFinder( @@ -159,6 +162,21 @@ public class FilterExpressionProcessor implements FilterProcessor { return listOfDataBlocksToScan; } + private String joinByteArray(byte[] bytes, char delimiter) { + String byteArrayAsString = ""; + if (null != bytes) { + for (int i = 0; i < bytes.length; i++) { + byteArrayAsString = byteArrayAsString + delimiter + bytes[i]; + } + if (byteArrayAsString.length() > 0) { + byteArrayAsString = byteArrayAsString.substring(1); + } + } else { + byteArrayAsString = null; + } + return byteArrayAsString; + } + /** * Get the map of required partitions * The value of "1" in BitSet represent the required partition http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerImpl.java index 145fc09..cb9495b 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerImpl.java @@ -30,6 +30,6 @@ public class FilterOptimizerImpl implements FilterOptimizer { } @Override public Expression optimizeFilter() { - return this.optimizeFilter(); + return filterExpression; } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java index 6d4e528..47ca7e1 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java @@ -145,7 +145,7 @@ public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverIm for (ExpressionResult result : listOfExpressionResults) { try { if (result.getString() == null) { - filterValuesList.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL.getBytes()); + filterValuesList.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY); continue; } filterValuesList.add(DataTypeUtil http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java b/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java index e78383d..21cf6c8 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java @@ -21,6 +21,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.math.BigDecimal; import java.nio.ByteBuffer; +import java.nio.charset.Charset; import java.util.Map; import org.apache.carbondata.common.logging.LogService; @@ -304,7 +305,8 @@ public abstract class AbstractScannedResult { j : rowMapping[pageCounter][j]); } - vector.putBytes(vectorOffset++, data.getBytes()); + vector.putBytes(vectorOffset++, + data.getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET))); } } } @@ -394,7 +396,8 @@ public abstract class AbstractScannedResult { int position = 0; for (int i = 0; i < this.noDictionaryColumnBlockIndexes.length; i++) { noDictionaryColumnsKeys[position++] = new String( - dimensionDataChunks[noDictionaryColumnBlockIndexes[i]][pageCounter].getChunkData(rowId)); + dimensionDataChunks[noDictionaryColumnBlockIndexes[i]][pageCounter].getChunkData(rowId), + Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)); } return noDictionaryColumnsKeys; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java index beef0c6..28d3f18 100644 --- a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java +++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java @@ -34,6 +34,7 @@ import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.datastore.impl.FileFactory; +import org.apache.carbondata.core.datastore.row.LoadStatusType; import org.apache.carbondata.core.fileoperations.AtomicFileOperations; import org.apache.carbondata.core.fileoperations.AtomicFileOperationsImpl; import org.apache.carbondata.core.fileoperations.FileWriteOperation; @@ -526,7 +527,9 @@ public class SegmentStatusManager { + "as the segment has been compacted."); continue; } - if (!CarbonCommonConstants.MARKED_FOR_DELETE.equals(loadMetadata.getLoadStatus())) { + if (!CarbonCommonConstants.MARKED_FOR_DELETE.equals(loadMetadata.getLoadStatus()) + && !LoadStatusType.IN_PROGRESS.getMessage().equals(loadMetadata.getLoadStatus()) + && !LoadStatusType.INSERT_OVERWRITE.getMessage().equals(loadMetadata.getLoadStatus())) { loadFound = true; updateSegmentMetadataDetails(loadMetadata); LOG.info("Info: " + http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java index 3e0a2cb..9abc58f 100644 --- a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java +++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java @@ -788,9 +788,6 @@ public class SegmentUpdateStatusManager { CarbonUpdateUtil.getRequiredFieldFromTID(completeBlockName, TupleIdEnum.BLOCK_ID))); String segmentId = CarbonUpdateUtil.getRequiredFieldFromTID(completeBlockName, TupleIdEnum.SEGMENT_ID); - CarbonTablePath carbonTablePath = CarbonStorePath - .getCarbonTablePath(absoluteTableIdentifier.getStorePath(), - absoluteTableIdentifier.getCarbonTableIdentifier()); SegmentUpdateDetails[] listOfSegmentUpdateDetailsArray = readLoadMetadata(); for (SegmentUpdateDetails block : listOfSegmentUpdateDetailsArray) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortInfoPreparator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortInfoPreparator.java b/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortInfoPreparator.java index 61ae6c2..8c7d922 100644 --- a/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortInfoPreparator.java +++ b/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortInfoPreparator.java @@ -123,8 +123,8 @@ public class CarbonDictionarySortInfoPreparator { // for new distinct values Iterator<String> distinctValue = distinctValues.iterator(); while (distinctValue.hasNext()) { - dictionarySortModels[surrogate - 1] = - createDictionarySortModel(surrogate, dataType, distinctValue.next().getBytes()); + dictionarySortModels[surrogate - 1] = createDictionarySortModel(surrogate, dataType, + distinctValue.next().getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET))); surrogate++; } return dictionarySortModels; http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java ---------------------------------------------------------------------- diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java index 631bc2c..1cdbb26 100644 --- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java +++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java @@ -20,6 +20,7 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; import java.io.Serializable; +import java.nio.charset.Charset; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -261,8 +262,10 @@ public class CarbonInputSplit extends FileSplit String filePath1 = this.getPath().getName(); String filePath2 = other.getPath().getName(); if (CarbonTablePath.isCarbonDataFile(filePath1)) { - byte[] firstTaskId = CarbonTablePath.DataFileUtil.getTaskNo(filePath1).getBytes(); - byte[] otherTaskId = CarbonTablePath.DataFileUtil.getTaskNo(filePath2).getBytes(); + byte[] firstTaskId = CarbonTablePath.DataFileUtil.getTaskNo(filePath1) + .getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)); + byte[] otherTaskId = CarbonTablePath.DataFileUtil.getTaskNo(filePath2) + .getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)); int compare = ByteUtil.compare(firstTaskId, otherTaskId); if (compare != 0) { return compare; http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java ---------------------------------------------------------------------- diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java index ed98871..4268ee2 100644 --- a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java +++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java @@ -79,7 +79,7 @@ class InMemoryBTreeIndex implements Index { FilterExpressionProcessor filterExpressionProcessor = new FilterExpressionProcessor(); - AbsoluteTableIdentifier identifier = null; + AbsoluteTableIdentifier identifier = AbsoluteTableIdentifier.from(segment.getPath(), "", ""); //for this segment fetch blocks matching filter in BTree List<DataRefNode> dataRefNodes = http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala index fb01b27..65f2ba2 100644 --- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala +++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala @@ -17,6 +17,7 @@ package org.apache.carbondata.spark.util +import java.nio.charset.Charset import java.text.SimpleDateFormat import org.apache.spark.sql._ @@ -109,7 +110,8 @@ object CarbonScalaUtil { case b: java.lang.Boolean => b.toString case s: java.lang.Short => s.toString case f: java.lang.Float => f.toString - case bs: Array[Byte] => new String(bs) + case bs: Array[Byte] => new String(bs, + Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)) case s: scala.collection.Seq[Any] => val delimiter = if (level == 1) { delimiterLevel1 http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/integration/spark2/src/main/scala/org/apache/spark/util/TableAPIUtil.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/util/TableAPIUtil.scala b/integration/spark2/src/main/scala/org/apache/spark/util/TableAPIUtil.scala index a57ab10..bc62902 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/util/TableAPIUtil.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/util/TableAPIUtil.scala @@ -33,7 +33,7 @@ object TableAPIUtil { def parseSchemaName(tableName: String): (String, String) = { if (tableName.contains(".")) { - val parts = tableName.split(".") + val parts = tableName.split("\\.") (parts(0), parts(1)) } else { ("default", tableName) http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/processing/src/main/java/org/apache/carbondata/processing/csvload/CSVInputFormat.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/csvload/CSVInputFormat.java b/processing/src/main/java/org/apache/carbondata/processing/csvload/CSVInputFormat.java index 3a6428d..c793126 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/csvload/CSVInputFormat.java +++ b/processing/src/main/java/org/apache/carbondata/processing/csvload/CSVInputFormat.java @@ -20,9 +20,11 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.Reader; +import java.nio.charset.Charset; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; +import org.apache.carbondata.core.constants.CarbonCommonConstants; import com.univocity.parsers.csv.CsvParser; import com.univocity.parsers.csv.CsvParserSettings; @@ -225,7 +227,8 @@ public class CSVInputFormat extends FileInputFormat<NullWritable, StringArrayWri filePosition = fileIn; inputStream = boundedInputStream; } - reader = new InputStreamReader(inputStream); + reader = new InputStreamReader(inputStream, + Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)); csvParser = new CsvParser(extractCsvParserSettings(job)); csvParser.beginParsing(reader); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/processing/src/main/java/org/apache/carbondata/processing/newflow/AbstractDataLoadProcessorStep.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/AbstractDataLoadProcessorStep.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/AbstractDataLoadProcessorStep.java index a5887e6..9e0aa02 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/newflow/AbstractDataLoadProcessorStep.java +++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/AbstractDataLoadProcessorStep.java @@ -56,7 +56,20 @@ public abstract class AbstractDataLoadProcessorStep { this.child = child; this.rowCounter = new AtomicLong(); this.closed = false; + } + + /** + * The output meta for this step. The data returns from this step is as per this meta. + * + */ + public abstract DataField[] getOutput(); + /** + * Initialization process for this step. + * + * @throws IOException + */ + public void initialize() throws IOException { if (LOGGER.isInfoEnabled()) { // This thread prints the rows processed in each step for every 10 seconds. new Thread() { @@ -76,19 +89,6 @@ public abstract class AbstractDataLoadProcessorStep { } /** - * The output meta for this step. The data returns from this step is as per this meta. - * - */ - public abstract DataField[] getOutput(); - - /** - * Initialization process for this step. - * - * @throws IOException - */ - public abstract void initialize() throws IOException; - - /** * Tranform the data as per the implementation. * * @return Array of Iterator with data. It can be processed parallel if implementation class wants http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java index 71e5727..4ed588d 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java +++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java @@ -84,6 +84,7 @@ public class CarbonRowDataWriterProcessorStepImpl extends AbstractDataLoadProces } @Override public void initialize() throws IOException { + super.initialize(); child.initialize(); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java index 62d6c94..c46ea25 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java +++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java @@ -61,6 +61,7 @@ public class DataConverterProcessorStepImpl extends AbstractDataLoadProcessorSte @Override public void initialize() throws IOException { + super.initialize(); child.initialize(); converters = new ArrayList<>(); badRecordLogger = createBadRecordLogger(configuration); http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java index c6f83ed..fe36f82 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java +++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java @@ -68,6 +68,7 @@ public class DataConverterProcessorWithBucketingStepImpl extends AbstractDataLoa @Override public void initialize() throws IOException { + super.initialize(); child.initialize(); converters = new ArrayList<>(); badRecordLogger = createBadRecordLogger(); http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterBatchProcessorStepImpl.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterBatchProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterBatchProcessorStepImpl.java index 46c1020..00cb302 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterBatchProcessorStepImpl.java +++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterBatchProcessorStepImpl.java @@ -55,6 +55,7 @@ public class DataWriterBatchProcessorStepImpl extends AbstractDataLoadProcessorS } @Override public void initialize() throws IOException { + super.initialize(); child.initialize(); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterProcessorStepImpl.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterProcessorStepImpl.java index 087b0c7..afb3062 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterProcessorStepImpl.java +++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterProcessorStepImpl.java @@ -62,6 +62,7 @@ public class DataWriterProcessorStepImpl extends AbstractDataLoadProcessorStep { } @Override public void initialize() throws IOException { + super.initialize(); child.initialize(); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/InputProcessorStepImpl.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/InputProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/InputProcessorStepImpl.java index 4a78e8f..cbeb20a 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/InputProcessorStepImpl.java +++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/InputProcessorStepImpl.java @@ -16,6 +16,7 @@ */ package org.apache.carbondata.processing.newflow.steps; +import java.io.IOException; import java.util.ArrayList; import java.util.Iterator; import java.util.List; @@ -32,7 +33,6 @@ import org.apache.carbondata.core.util.CarbonProperties; import org.apache.carbondata.processing.newflow.AbstractDataLoadProcessorStep; import org.apache.carbondata.processing.newflow.CarbonDataLoadConfiguration; import org.apache.carbondata.processing.newflow.DataField; -import org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException; import org.apache.carbondata.processing.newflow.parser.RowParser; import org.apache.carbondata.processing.newflow.parser.impl.RowParserImpl; import org.apache.carbondata.processing.newflow.row.CarbonRowBatch; @@ -61,7 +61,8 @@ public class InputProcessorStepImpl extends AbstractDataLoadProcessorStep { return configuration.getDataFields(); } - @Override public void initialize() throws CarbonDataLoadingException { + @Override public void initialize() throws IOException { + super.initialize(); rowParser = new RowParserImpl(getOutput(), configuration); executorService = Executors.newCachedThreadPool(); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/SortProcessorStepImpl.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/SortProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/SortProcessorStepImpl.java index 987cb15..0b93b7a 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/SortProcessorStepImpl.java +++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/SortProcessorStepImpl.java @@ -49,6 +49,7 @@ public class SortProcessorStepImpl extends AbstractDataLoadProcessorStep { @Override public void initialize() throws IOException { + super.initialize(); child.initialize(); SortParameters sortParameters = SortParameters.createSortParameters(configuration); sorter = SorterFactory.createSorter(configuration, rowCounter); http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/processing/src/main/java/org/apache/carbondata/processing/store/writer/v2/CarbonFactDataWriterImplV2.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v2/CarbonFactDataWriterImplV2.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v2/CarbonFactDataWriterImplV2.java index 82e83d5..c835332 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v2/CarbonFactDataWriterImplV2.java +++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v2/CarbonFactDataWriterImplV2.java @@ -19,6 +19,7 @@ package org.apache.carbondata.processing.store.writer.v2; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; +import java.nio.charset.Charset; import java.util.ArrayList; import java.util.List; @@ -107,7 +108,8 @@ public class CarbonFactDataWriterImplV2 extends CarbonFactDataWriterImplV1 { try { if (fileChannel.size() == 0) { ColumnarFormatVersion version = CarbonProperties.getInstance().getFormatVersion(); - byte[] header = (CarbonCommonConstants.CARBON_DATA_VERSION_HEADER + version).getBytes(); + byte[] header = (CarbonCommonConstants.CARBON_DATA_VERSION_HEADER + version).getBytes( + Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)); ByteBuffer buffer = ByteBuffer.allocate(header.length); buffer.put(header); buffer.rewind();
