Repository: carbondata Updated Branches: refs/heads/master 697eee3de -> 50ecb83a2
[CARBONDATA-3114]Remove Null Values for a Dictionary_Include Timestamp column for Range Filters Problem: Null Values are not removed in case of RangeFilters, if column is a dictionary and no_inverted_index timestamp column. Solution: Remove NULL values in case of RangeFilters for such dictionary and no_inverted_index timestamp column. This closes #2937 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/50ecb83a Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/50ecb83a Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/50ecb83a Branch: refs/heads/master Commit: 50ecb83a264ab6512ebade0580e3288295452966 Parents: 697eee3 Author: Indhumathi27 <[email protected]> Authored: Wed Nov 21 15:21:49 2018 +0530 Committer: kumarvishal09 <[email protected]> Committed: Thu Nov 22 16:32:15 2018 +0530 ---------------------------------------------------------------------- .../carbondata/core/scan/filter/FilterUtil.java | 23 +++++++++++++++ .../executer/RangeValueFilterExecuterImpl.java | 21 ++------------ .../RowLevelRangeGrtThanFiterExecuterImpl.java | 8 +++++- ...elRangeGrtrThanEquaToFilterExecuterImpl.java | 8 +++++- ...velRangeLessThanEqualFilterExecuterImpl.java | 20 ++++--------- ...RowLevelRangeLessThanFilterExecuterImpl.java | 20 ++++--------- .../src/test/resources/data_timestamp.csv | 10 +++++++ ...estampDataTypeDirectDictionaryTestCase.scala | 30 ++++++++++++++++++++ 8 files changed, 89 insertions(+), 51 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/50ecb83a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java index 06672f5..286f68f 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java @@ -52,6 +52,8 @@ import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage; import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk; import org.apache.carbondata.core.keygenerator.KeyGenException; import org.apache.carbondata.core.keygenerator.KeyGenerator; +import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator; +import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory; import org.apache.carbondata.core.keygenerator.factory.KeyGeneratorFactory; import org.apache.carbondata.core.keygenerator.mdkey.MultiDimKeyVarLengthGenerator; import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; @@ -2247,4 +2249,25 @@ public final class FilterUtil { } } + /** + * This method is used to get default null values for a direct dictionary column + * @param currentBlockDimension + * @param segmentProperties + * @return + */ + public static byte[] getDefaultNullValue(CarbonDimension currentBlockDimension, + SegmentProperties segmentProperties) { + byte[] defaultValue = null; + DirectDictionaryGenerator directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory + .getDirectDictionaryGenerator(currentBlockDimension.getDataType()); + int key = directDictionaryGenerator.generateDirectSurrogateKey(null); + if (currentBlockDimension.isSortColumn()) { + defaultValue = FilterUtil + .getMaskKey(key, currentBlockDimension, segmentProperties.getSortColumnsGenerator()); + } else { + defaultValue = ByteUtil.toXorBytes(key); + } + return defaultValue; + } + } http://git-wip-us.apache.org/repos/asf/carbondata/blob/50ecb83a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java index e84e82d..bcae001 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java @@ -24,8 +24,6 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.datastore.block.SegmentProperties; import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage; import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk; -import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator; -import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory; import org.apache.carbondata.core.metadata.datatype.DataTypes; import org.apache.carbondata.core.metadata.encoder.Encoding; import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension; @@ -55,10 +53,6 @@ public class RangeValueFilterExecuterImpl implements FilterExecuter { private byte[][] filterRangesValues; private SegmentProperties segmentProperties; private boolean isDefaultValuePresentInFilter; - /** - * it has index at which given dimension is stored in file - */ - private int dimensionChunkIndex; /** * flag to check whether the filter dimension is present in current block list of dimensions. @@ -106,8 +100,6 @@ public class RangeValueFilterExecuterImpl implements FilterExecuter { segmentProperties.getDimensionFromCurrentBlock(dimColEvaluatorInfo.getDimension()); if (null != dimensionFromCurrentBlock) { dimColEvaluatorInfo.setColumnIndex(dimensionFromCurrentBlock.getOrdinal()); - this.dimensionChunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping() - .get(dimensionFromCurrentBlock.getOrdinal()); isDimensionPresentInCurrentBlock = true; } } @@ -656,17 +648,8 @@ public class RangeValueFilterExecuterImpl implements FilterExecuter { } else { byte[] defaultValue = null; if (dimColEvaluatorInfo.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) { - DirectDictionaryGenerator directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory - .getDirectDictionaryGenerator(dimColEvaluatorInfo.getDimension().getDataType()); - int key = directDictionaryGenerator.generateDirectSurrogateKey(null); - CarbonDimension currentBlockDimension = - segmentProperties.getDimensions().get(dimensionChunkIndex); - if (currentBlockDimension.isSortColumn()) { - defaultValue = FilterUtil.getMaskKey(key, currentBlockDimension, - this.segmentProperties.getSortColumnsGenerator()); - } else { - defaultValue = ByteUtil.toXorBytes(key); - } + defaultValue = + FilterUtil.getDefaultNullValue(dimColEvaluatorInfo.getDimension(), segmentProperties); } else { if (dimColEvaluatorInfo.getDimension().getDataType() == DataTypes.STRING) { defaultValue = CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY; http://git-wip-us.apache.org/repos/asf/carbondata/blob/50ecb83a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java index 06dc3c4..d2c4b05 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java @@ -450,10 +450,16 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute byte[] defaultValue = null; if (dimColEvaluatorInfoList.get(0).getDimension().getDataType() == DataTypes.STRING) { defaultValue = CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY; + } else if (dimColEvaluatorInfoList.get(0).getDimension() + .hasEncoding(Encoding.DIRECT_DICTIONARY)) { + defaultValue = FilterUtil + .getDefaultNullValue(dimColEvaluatorInfoList.get(0).getDimension(), segmentProperties); } else if (!dimensionColumnPage.isAdaptiveEncoded()) { defaultValue = CarbonCommonConstants.EMPTY_BYTE_ARRAY; } - if (dimensionColumnPage.isNoDicitionaryColumn()) { + if (dimensionColumnPage.isNoDicitionaryColumn() || ( + dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY) + && dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY))) { FilterUtil.removeNullValues(dimensionColumnPage, bitSet, defaultValue); } return bitSet; http://git-wip-us.apache.org/repos/asf/carbondata/blob/50ecb83a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java index 02c587e..cf31033 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java @@ -454,10 +454,16 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte byte[] defaultValue = null; if (dimColEvaluatorInfoList.get(0).getDimension().getDataType() == DataTypes.STRING) { defaultValue = CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY; + } else if (dimColEvaluatorInfoList.get(0).getDimension() + .hasEncoding(Encoding.DIRECT_DICTIONARY)) { + defaultValue = FilterUtil + .getDefaultNullValue(dimColEvaluatorInfoList.get(0).getDimension(), segmentProperties); } else if (!dimensionColumnPage.isAdaptiveEncoded()) { defaultValue = CarbonCommonConstants.EMPTY_BYTE_ARRAY; } - if (dimensionColumnPage.isNoDicitionaryColumn()) { + if (dimensionColumnPage.isNoDicitionaryColumn() || ( + dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY) + && dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY))) { FilterUtil.removeNullValues(dimensionColumnPage, bitSet, defaultValue); } return bitSet; http://git-wip-us.apache.org/repos/asf/carbondata/blob/50ecb83a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java index ac9661e..8ea6e0d 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java @@ -26,8 +26,6 @@ import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage; import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk; import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk; import org.apache.carbondata.core.datastore.page.ColumnPage; -import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator; -import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory; import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; import org.apache.carbondata.core.metadata.datatype.DataType; import org.apache.carbondata.core.metadata.datatype.DataTypes; @@ -426,18 +424,8 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter int numerOfRows) { byte[] defaultValue = null; if (dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) { - DirectDictionaryGenerator directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory - .getDirectDictionaryGenerator( - dimColEvaluatorInfoList.get(0).getDimension().getDataType()); - int key = directDictionaryGenerator.generateDirectSurrogateKey(null); - CarbonDimension currentBlockDimension = - segmentProperties.getDimensions().get(dimensionChunkIndex[0]); - if (currentBlockDimension.isSortColumn()) { - defaultValue = FilterUtil.getMaskKey(key, currentBlockDimension, - this.segmentProperties.getSortColumnsGenerator()); - } else { - defaultValue = ByteUtil.toXorBytes(key); - } + defaultValue = FilterUtil + .getDefaultNullValue(dimColEvaluatorInfoList.get(0).getDimension(), segmentProperties); } else if (dimColEvaluatorInfoList.get(0).getDimension().getDataType() != DataTypes.STRING) { defaultValue = CarbonCommonConstants.EMPTY_BYTE_ARRAY; } @@ -452,7 +440,9 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter if (dimColEvaluatorInfoList.get(0).getDimension().getDataType() == DataTypes.STRING) { defaultValue = CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY; } - if (dimensionColumnPage.isNoDicitionaryColumn()) { + if (dimensionColumnPage.isNoDicitionaryColumn() || ( + dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY) + && dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY))) { FilterUtil.removeNullValues(dimensionColumnPage, bitSet, defaultValue); } return bitSet; http://git-wip-us.apache.org/repos/asf/carbondata/blob/50ecb83a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFilterExecuterImpl.java index 644cf02..df1afc4 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFilterExecuterImpl.java @@ -26,8 +26,6 @@ import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage; import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk; import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk; import org.apache.carbondata.core.datastore.page.ColumnPage; -import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator; -import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory; import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; import org.apache.carbondata.core.metadata.datatype.DataType; import org.apache.carbondata.core.metadata.datatype.DataTypes; @@ -422,18 +420,8 @@ public class RowLevelRangeLessThanFilterExecuterImpl extends RowLevelFilterExecu int numerOfRows) { byte[] defaultValue = null; if (dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) { - DirectDictionaryGenerator directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory - .getDirectDictionaryGenerator( - dimColEvaluatorInfoList.get(0).getDimension().getDataType()); - int key = directDictionaryGenerator.generateDirectSurrogateKey(null); - CarbonDimension currentBlockDimension = - segmentProperties.getDimensions().get(dimensionChunkIndex[0]); - if (currentBlockDimension.isSortColumn()) { - defaultValue = FilterUtil.getMaskKey(key, currentBlockDimension, - this.segmentProperties.getSortColumnsGenerator()); - } else { - defaultValue = ByteUtil.toXorBytes(key); - } + defaultValue = FilterUtil + .getDefaultNullValue(dimColEvaluatorInfoList.get(0).getDimension(), segmentProperties); } else if (dimColEvaluatorInfoList.get(0).getDimension().getDataType() != DataTypes.STRING) { defaultValue = CarbonCommonConstants.EMPTY_BYTE_ARRAY; } @@ -448,7 +436,9 @@ public class RowLevelRangeLessThanFilterExecuterImpl extends RowLevelFilterExecu if (dimColEvaluatorInfoList.get(0).getDimension().getDataType() == DataTypes.STRING) { defaultValue = CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY; } - if (dimensionColumnPage.isNoDicitionaryColumn()) { + if (dimensionColumnPage.isNoDicitionaryColumn() || ( + dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY) + && dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY))) { FilterUtil.removeNullValues(dimensionColumnPage, bitSet, defaultValue); } return bitSet; http://git-wip-us.apache.org/repos/asf/carbondata/blob/50ecb83a/integration/spark-common-test/src/test/resources/data_timestamp.csv ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/resources/data_timestamp.csv b/integration/spark-common-test/src/test/resources/data_timestamp.csv new file mode 100644 index 0000000..d11c69a --- /dev/null +++ b/integration/spark-common-test/src/test/resources/data_timestamp.csv @@ -0,0 +1,10 @@ +col +2014-01-01 18:00:00 +2014-01-02 18:00:00 +2014-01-03 18:00:00 + +2014-01-03 18:00:00 +0 +2014-01-03 18:00:00 + +2014-01-03 18:00:00 http://git-wip-us.apache.org/repos/asf/carbondata/blob/50ecb83a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala index b8d057c..6c8ab53 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala @@ -141,9 +141,39 @@ class TimestampDataTypeDirectDictionaryTest extends QueryTest with BeforeAndAfte ) } + test("test timestamp with dictionary include and no_inverted index") { + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.CARBON_PUSH_ROW_FILTERS_FOR_VECTOR, "true") + sql("drop table if exists test_timestamp") + sql("drop table if exists test_timestamp_hive") + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy-MM-dd HH:mm:ss") + sql( + "create table test_timestamp(col timestamp) stored by 'carbondata' tblproperties" + + "('no_inverted_index'='col','dictionary_include'='col')") + val csvFilePath = s"$resourcesPath/data_timestamp.csv" + sql( + "load data inpath '" + csvFilePath + + "' into table test_timestamp options('delimiter'='=','quotechar'=''," + + "'bad_records_action'='force','fileheader'='col')") + sql( + "create table test_timestamp_hive(col timestamp) row format delimited fields terminated by " + + "','") + sql("load data inpath '" + csvFilePath + "' into table test_timestamp_hive ") + checkAnswer(sql( + "select col from test_timestamp where col not between '2014-01-01 18:00:00' and '0'"), + sql("select col from test_timestamp_hive where col not between '2014-01-01 18:00:00' and " + + "'0'")) + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.CARBON_PUSH_ROW_FILTERS_FOR_VECTOR, + CarbonCommonConstants.CARBON_PUSH_ROW_FILTERS_FOR_VECTOR_DEFAULT) + } + override def afterAll { sql("drop table directDictionaryTable") sql("drop table directDictionaryTable_hive") + sql("drop table if exists test_timestamp") + sql("drop table if exists test_timestamp_hive") CarbonProperties.getInstance() .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
