Repository: incubator-carbondata Updated Branches: refs/heads/12-dev 47b2fdd90 -> 222a80762
fix sort_columns issue Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/14c0a8b4 Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/14c0a8b4 Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/14c0a8b4 Branch: refs/heads/12-dev Commit: 14c0a8b412d52c5fd03050df8eeb61f9f72afc07 Parents: 47b2fdd Author: QiangCai <[email protected]> Authored: Sat Apr 8 22:10:17 2017 +0530 Committer: QiangCai <[email protected]> Committed: Tue Apr 11 09:52:55 2017 +0530 ---------------------------------------------------------------------- .../executer/RowLevelFilterExecuterImpl.java | 3 ++- .../RowLevelRangeFilterResolverImpl.java | 5 +++- .../apache/carbondata/core/util/ByteUtil.java | 28 +++++++++++--------- .../carbondata/core/util/DataTypeUtil.java | 2 +- .../testsuite/sortcolumns/TestSortColumns.scala | 11 ++++++-- .../CarbonRowDataWriterProcessorStepImpl.java | 4 +-- 6 files changed, 33 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/14c0a8b4/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java index d92b7d4..22490ac 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java @@ -246,7 +246,8 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter { memberBytes = null; } record[dimColumnEvaluatorInfo.getRowIndex()] = DataTypeUtil - .getDataBasedOnDataType(memberBytes, dimColumnEvaluatorInfo.getDimension()); + .getDataBasedOnDataTypeForNoDictionaryColumn(memberBytes, + dimColumnEvaluatorInfo.getDimension().getDataType()); } else { continue; } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/14c0a8b4/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java index 2a1af65..6d4e528 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java @@ -43,6 +43,7 @@ import org.apache.carbondata.core.scan.filter.intf.FilterExecuterType; import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo; import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo; import org.apache.carbondata.core.util.ByteUtil; +import org.apache.carbondata.core.util.DataTypeUtil; public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverImpl { @@ -147,7 +148,9 @@ public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverIm filterValuesList.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL.getBytes()); continue; } - filterValuesList.add(result.getString().getBytes()); + filterValuesList.add(DataTypeUtil + .getBytesBasedOnDataTypeForNoDictionaryColumn(result.getString(), + result.getDataType())); } catch (FilterIllegalMemberException e) { // Any invalid member while evaluation shall be ignored, system will log the // error only once since all rows the evaluation happens so inorder to avoid http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/14c0a8b4/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java index 84a8634..1aa2fa1 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java @@ -441,6 +441,7 @@ public final class ByteUtil { * @return */ public static byte[] toBytes(short val) { + val = (short)(val ^ Short.MIN_VALUE); byte[] b = new byte[SIZEOF_SHORT]; b[1] = (byte) val; val >>= 8; @@ -460,20 +461,21 @@ public final class ByteUtil { if (length != SIZEOF_SHORT || offset + length > bytes.length) { throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_SHORT); } + short n = 0; if (CarbonUnsafe.unsafe != null) { if (CarbonUnsafe.ISLITTLEENDIAN) { - return Short.reverseBytes( + n = Short.reverseBytes( CarbonUnsafe.unsafe.getShort(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET)); } else { - return CarbonUnsafe.unsafe.getShort(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET); + n = CarbonUnsafe.unsafe.getShort(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET); } } else { - short n = 0; + n ^= bytes[offset] & 0xFF; n <<= 8; n ^= bytes[offset + 1] & 0xFF; - return n; } + return (short)(n ^ Short.MIN_VALUE); } /** @@ -483,6 +485,7 @@ public final class ByteUtil { * @return */ public static byte[] toBytes(int val) { + val = val ^ Integer.MIN_VALUE; byte[] b = new byte[4]; for (int i = 3; i > 0; i--) { b[i] = (byte) val; @@ -504,21 +507,21 @@ public final class ByteUtil { if (length != SIZEOF_INT || offset + length > bytes.length) { throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_INT); } + int n = 0; if (CarbonUnsafe.unsafe != null) { if (CarbonUnsafe.ISLITTLEENDIAN) { - return Integer.reverseBytes( + n = Integer.reverseBytes( CarbonUnsafe.unsafe.getInt(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET)); } else { - return CarbonUnsafe.unsafe.getInt(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET); + n = CarbonUnsafe.unsafe.getInt(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET); } } else { - int n = 0; for (int i = offset; i < (offset + length); i++) { n <<= 8; n ^= bytes[i] & 0xFF; } - return n; } + return n ^ Integer.MIN_VALUE; } /** @@ -550,6 +553,7 @@ public final class ByteUtil { * @return */ public static byte[] toBytes(long val) { + val = val ^ Long.MIN_VALUE; byte[] b = new byte[8]; for (int i = 7; i > 0; i--) { b[i] = (byte) val; @@ -566,21 +570,21 @@ public final class ByteUtil { if (length != SIZEOF_LONG || offset + length > bytes.length) { throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_LONG); } + long l = 0; if (CarbonUnsafe.unsafe != null) { if (CarbonUnsafe.ISLITTLEENDIAN) { - return Long.reverseBytes( + l = Long.reverseBytes( CarbonUnsafe.unsafe.getLong(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET)); } else { - return CarbonUnsafe.unsafe.getLong(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET); + l = CarbonUnsafe.unsafe.getLong(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET); } } else { - long l = 0; for (int i = offset; i < offset + length; i++) { l <<= 8; l ^= bytes[i] & 0xFF; } - return l; } + return l ^ Long.MIN_VALUE; } /** http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/14c0a8b4/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java index 2e16b0f..80c9e72 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java @@ -324,7 +324,7 @@ public final class DataTypeUtil { } public static byte[] getBytesBasedOnDataTypeForNoDictionaryColumn(String dimensionValue, - DataType actualDataType) throws Throwable { + DataType actualDataType) { switch (actualDataType) { case STRING: return ByteUtil.toBytes(dimensionValue); http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/14c0a8b4/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala index 88b0421..afdca21 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala @@ -220,13 +220,20 @@ class TestSortColumns extends QueryTest with BeforeAndAfterAll { sql("CREATE TABLE unsortedtable_offheap_inmemory (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')") sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_offheap_inmemory OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""") checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno = 11"), sql("select * from origintable1 where empno = 11")) + checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno < 15 order by empno"), sql("select * from origintable1 where empno < 15 order by empno")) + checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno <= 15 order by empno"), sql("select * from origintable1 where empno <= 15 order by empno")) + checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno > 15 order by empno"), sql("select * from origintable1 where empno > 15 order by empno")) + checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno >= 15 order by empno"), sql("select * from origintable1 where empno >= 15 order by empno")) + checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno <> 15 order by empno"), sql("select * from origintable1 where empno <> 15 order by empno")) + checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno in (15, 16, 17) order by empno"), sql("select * from origintable1 where empno in (15, 16, 17) order by empno")) + checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno is null"), sql("select * from origintable1 where empno is null order by empno")) + checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno is not null"), sql("select * from origintable1 where empno is not null order by empno")) checkAnswer(sql("select * from unsortedtable_offheap_inmemory order by empno"), sql("select * from origintable1 order by empno")) } finally { defaultLoadingProperties } } - - + override def afterAll = { dropTable } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/14c0a8b4/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java index 929a09e..c50f335 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java +++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java @@ -156,9 +156,7 @@ public class CarbonRowDataWriterProcessorStepImpl extends AbstractDataLoadProces private void doExecute(Iterator<CarbonRowBatch> iterator, int partitionId, int iteratorIndex) { String storeLocation = getStoreLocation(tableIdentifier, String.valueOf(partitionId)); CarbonFactDataHandlerModel model = CarbonFactDataHandlerModel - .createCarbonFactDataHandlerModel(configuration, storeLocation, partitionId, 0); - model.getCarbonDataFileAttributes() - .setFactTimeStamp(model.getCarbonDataFileAttributes().getFactTimeStamp() + iteratorIndex); + .createCarbonFactDataHandlerModel(configuration, storeLocation, partitionId, iteratorIndex); CarbonFactHandler dataHandler = null; boolean rowsNotExist = true; while (iterator.hasNext()) {
