Repository: carbondata Updated Branches: refs/heads/master f4a58c545 -> 3a4b88138
[CARBONDATA-2589][CARBONDATA-2590][CARBONDATA-2602]Local dictionary query Support Supported Non filter query for local dictionary Supported Filter query on local dictionary Supported Query on complex column for primitive type local dictionary columns Local Dictionary support on Varchar columns Supported Vector reader on local dictionary Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/4935cb18 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/4935cb18 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/4935cb18 Branch: refs/heads/master Commit: 4935cb18841b6965ce012e264d66e851e4bf9e8a Parents: f4a58c5 Author: akashrn5 <[email protected]> Authored: Thu Jul 5 12:01:19 2018 +0530 Committer: kunal642 <[email protected]> Committed: Tue Jul 10 11:04:46 2018 +0530 ---------------------------------------------------------------------- .../core/metadata/schema/table/CarbonTable.java | 2 +- .../carbondata/core/scan/filter/FilterUtil.java | 125 ++++++++++++++++++- .../executer/ExcludeFilterExecuterImpl.java | 22 ++-- .../executer/IncludeFilterExecuterImpl.java | 30 +++-- .../executer/RangeValueFilterExecuterImpl.java | 41 +++++- .../RowLevelRangeGrtThanFiterExecuterImpl.java | 29 ++++- ...elRangeGrtrThanEquaToFilterExecuterImpl.java | 29 ++++- ...velRangeLessThanEqualFilterExecuterImpl.java | 29 ++++- ...RowLevelRangeLessThanFilterExecuterImpl.java | 29 ++++- .../apache/carbondata/core/util/CarbonUtil.java | 14 +-- .../TestPageLevelDictionary.java | 10 +- .../executer/ExcludeFilterExecuterImplTest.java | 2 +- .../executer/IncludeFilterExecuterImplTest.java | 2 +- .../carbondata/core/util/CarbonUtilTest.java | 4 +- .../TableLevelCompactionOptionTest.scala | 2 +- .../testsuite/datamap/DataMapWriterSuite.scala | 2 +- .../CarbonGetTableDetailComandTestCase.scala | 4 +- 17 files changed, 317 insertions(+), 59 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/4935cb18/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java index be42f3f..80853f9 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java @@ -486,7 +486,7 @@ public class CarbonTable implements Serializable { * @return */ public boolean isLocalDictionaryEnabled() { - return false; + return isLocalDictionaryEnabled; } /** http://git-wip-us.apache.org/repos/asf/carbondata/blob/4935cb18/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java index d23e2d6..a657b91 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java @@ -50,8 +50,10 @@ import org.apache.carbondata.core.constants.CarbonV3DataFormatConstants; import org.apache.carbondata.core.datastore.IndexKey; import org.apache.carbondata.core.datastore.block.SegmentProperties; import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage; +import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk; import org.apache.carbondata.core.keygenerator.KeyGenException; import org.apache.carbondata.core.keygenerator.KeyGenerator; +import org.apache.carbondata.core.keygenerator.factory.KeyGeneratorFactory; import org.apache.carbondata.core.keygenerator.mdkey.MultiDimKeyVarLengthGenerator; import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; import org.apache.carbondata.core.metadata.CarbonMetadata; @@ -69,6 +71,7 @@ import org.apache.carbondata.core.scan.expression.ColumnExpression; import org.apache.carbondata.core.scan.expression.Expression; import org.apache.carbondata.core.scan.expression.ExpressionResult; import org.apache.carbondata.core.scan.expression.LiteralExpression; +import org.apache.carbondata.core.scan.expression.conditional.ConditionalExpression; import org.apache.carbondata.core.scan.expression.conditional.InExpression; import org.apache.carbondata.core.scan.expression.conditional.ListExpression; import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException; @@ -99,6 +102,7 @@ import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf; import org.apache.carbondata.core.scan.filter.resolver.RowLevelFilterResolverImpl; import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo; import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo; +import org.apache.carbondata.core.scan.result.vector.CarbonDictionary; import org.apache.carbondata.core.util.BitSetGroup; import org.apache.carbondata.core.util.ByteUtil; import org.apache.carbondata.core.util.CarbonProperties; @@ -564,7 +568,8 @@ public final class FilterUtil { boolean isExcludeFilterNeedsToApply = false; if (isIncludeFilter) { isExcludeFilterNeedsToApply = - isExcludeFilterNeedsToApply(forwardDictionary, surrogates.size()); + isExcludeFilterNeedsToApply(forwardDictionary.getDictionaryChunks().getSize(), + surrogates.size()); } Collections.sort(surrogates); ColumnFilterInfo columnFilterInfo = null; @@ -581,9 +586,9 @@ public final class FilterUtil { return columnFilterInfo; } - private static boolean isExcludeFilterNeedsToApply(Dictionary forwardDictionary, + public static boolean isExcludeFilterNeedsToApply(int dictionarySize, int size) { - if ((size * 100) / forwardDictionary.getDictionaryChunks().getSize() >= 60) { + if ((size * 100) / dictionarySize >= 60) { LOGGER.info("Applying CBO to convert include filter to exclude filter."); return true; } @@ -1839,4 +1844,118 @@ public final class FilterUtil { } } } + + public static byte[][] getEncodedFilterValues(CarbonDictionary dictionary, + byte[][] actualFilterValues) { + if (null == dictionary) { + return actualFilterValues; + } + KeyGenerator keyGenerator = KeyGeneratorFactory.getKeyGenerator(new int[] { 100000 }); + List<byte[]> encodedFilters = new ArrayList<>(); + for (byte[] actualFilter : actualFilterValues) { + for (int i = 1; i < dictionary.getDictionaryValues().length; i++) { + if (ByteUtil.UnsafeComparer.INSTANCE + .compareTo(actualFilter, dictionary.getDictionaryValues()[i]) + == 0) { + try { + encodedFilters.add(keyGenerator.generateKey(new int[] { i })); + } catch (KeyGenException e) { + //do nothing + } + break; + } + } + } + return getSortedEncodedFilters(encodedFilters); + } + + private static byte[][] getSortedEncodedFilters(List<byte[]> encodedFilters) { + java.util.Comparator<byte[]> filterNoDictValueComaparator = new java.util.Comparator<byte[]>() { + @Override public int compare(byte[] filterMember1, byte[] filterMember2) { + return ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterMember1, filterMember2); + } + }; + Collections.sort(encodedFilters, filterNoDictValueComaparator); + return encodedFilters.toArray(new byte[encodedFilters.size()][]); + } + + private static BitSet getIncludeDictionaryValues(Expression expression, + CarbonDictionary dictionary) throws FilterUnsupportedException { + ConditionalExpression conExp = (ConditionalExpression) expression; + ColumnExpression columnExpression = conExp.getColumnList().get(0); + BitSet includeFilterBitSet = new BitSet(); + for (int i = 2; i < dictionary.getDictionaryValues().length; i++) { + try { + RowIntf row = new RowImpl(); + String stringValue = new String(dictionary.getDictionaryValues()[i], + Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)); + row.setValues(new Object[] { DataTypeUtil.getDataBasedOnDataType(stringValue, + columnExpression.getCarbonColumn().getDataType()) }); + Boolean rslt = expression.evaluate(row).getBoolean(); + if (null != rslt) { + if (rslt) { + includeFilterBitSet.set(i); + } + } + } catch (FilterIllegalMemberException e) { + LOGGER.debug(e.getMessage()); + } + } + return includeFilterBitSet; + } + + public static byte[][] getEncodedFilterValues(BitSet includeDictValues, int dictSize, + boolean useExclude) { + KeyGenerator keyGenerator = KeyGeneratorFactory + .getKeyGenerator(new int[] { CarbonCommonConstants.LOCAL_DICTIONARY_MAX }); + List<byte[]> encodedFilterValues = new ArrayList<>(); + int[] dummy = new int[1]; + if (!useExclude) { + try { + for (int i = includeDictValues.nextSetBit(0); + i >= 0; i = includeDictValues.nextSetBit(i + 1)) { + dummy[0] = i; + encodedFilterValues.add(keyGenerator.generateKey(dummy)); + } + } catch (KeyGenException e) { + // do nothing + } + return encodedFilterValues.toArray(new byte[encodedFilterValues.size()][]); + } else { + try { + for (int i = 1; i < dictSize; i++) { + if (!includeDictValues.get(i)) { + dummy[0] = i; + encodedFilterValues.add(keyGenerator.generateKey(dummy)); + } + } + } catch (KeyGenException e) { + // do nothing + } + } + return getSortedEncodedFilters(encodedFilterValues); + } + + public static FilterExecuter getFilterExecutorForLocalDictionary( + DimensionRawColumnChunk rawColumnChunk, Expression exp, boolean isNaturalSorted) { + BitSet includeDictionaryValues; + try { + includeDictionaryValues = + FilterUtil.getIncludeDictionaryValues(exp, rawColumnChunk.getLocalDictionary()); + } catch (FilterUnsupportedException e) { + throw new RuntimeException(e); + } + boolean isExclude = includeDictionaryValues.cardinality() > 1 && FilterUtil + .isExcludeFilterNeedsToApply(rawColumnChunk.getLocalDictionary().getDictionarySize(), + includeDictionaryValues.cardinality()); + byte[][] encodedFilterValues = FilterUtil.getEncodedFilterValues(includeDictionaryValues, + rawColumnChunk.getLocalDictionary().getDictionaryValues().length, isExclude); + FilterExecuter filterExecuter; + if (!isExclude) { + filterExecuter = new IncludeFilterExecuterImpl(encodedFilterValues, isNaturalSorted); + } else { + filterExecuter = new ExcludeFilterExecuterImpl(encodedFilterValues, isNaturalSorted); + } + return filterExecuter; + } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/4935cb18/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java index a608c8e..7646550 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java @@ -53,6 +53,12 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter { */ private boolean isNaturalSorted = false; + private byte[][] filterValues; + + public ExcludeFilterExecuterImpl(byte[][] filterValues, boolean isNaturalSorted) { + this.filterValues = filterValues; + this.isNaturalSorted = isNaturalSorted; + } public ExcludeFilterExecuterImpl(DimColumnResolvedFilterInfo dimColEvaluatorInfo, MeasureColumnResolvedFilterInfo msrColumnEvaluatorInfo, SegmentProperties segmentProperties, boolean isMeasure) { @@ -96,6 +102,9 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter { rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]; DimensionColumnPage[] dimensionColumnPages = dimensionRawColumnChunk.decodeAllColumnPages(); + filterValues = FilterUtil + .getEncodedFilterValues(dimensionRawColumnChunk.getLocalDictionary(), + dimColumnExecuterInfo.filterKeysForExclude); BitSetGroup bitSetGroup = new BitSetGroup(dimensionRawColumnChunk.getPagesCount()); for (int i = 0; i < dimensionColumnPages.length; i++) { BitSet bitSet = getFilteredIndexes(dimensionColumnPages[i], @@ -285,11 +294,11 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter { * @param pageNumber * @return filtered indexes bitset */ - private BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage, + protected BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage, int numberOfRows, boolean useBitsetPipeLine, BitSetGroup prvBitSetGroup, int pageNumber) { // check whether applying filtered based on previous bitset will be optimal if (CarbonUtil.usePreviousFilterBitsetGroup(useBitsetPipeLine, prvBitSetGroup, pageNumber, - dimColumnExecuterInfo.getExcludeFilterKeys().length)) { + filterValues.length)) { return getFilteredIndexesUisngPrvBitset(dimensionColumnPage, prvBitSetGroup, pageNumber); } else { return getFilteredIndexes(dimensionColumnPage, numberOfRows); @@ -319,7 +328,6 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter { } BitSet bitSet = new BitSet(); bitSet.or(prvPageBitSet); - byte[][] filterKeys = dimColumnExecuterInfo.getExcludeFilterKeys(); int compareResult = 0; // if dimension data was natural sorted then get the index from previous bitset // and use the same in next column data, otherwise use the inverted index reverse @@ -327,7 +335,7 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter { for (int index = prvPageBitSet.nextSetBit(0); index >= 0; index = prvPageBitSet.nextSetBit(index + 1)) { compareResult = CarbonUtil - .isFilterPresent(filterKeys, dimensionColumnPage, 0, filterKeys.length - 1, index); + .isFilterPresent(filterValues, dimensionColumnPage, 0, filterValues.length - 1, index); if (compareResult != 0) { bitSet.set(index); } else { @@ -340,7 +348,7 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter { for (int index = prvPageBitSet.nextSetBit(0); index >= 0; index = prvPageBitSet.nextSetBit(index + 1)) { compareResult = CarbonUtil - .isFilterPresent(filterKeys, dimensionColumnPage, 0, filterKeys.length - 1, + .isFilterPresent(filterValues, dimensionColumnPage, 0, filterValues.length - 1, dimensionColumnPage.getInvertedReverseIndex(index)); if (compareResult != 0) { bitSet.set(index); @@ -359,7 +367,6 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter { BitSet bitSet = new BitSet(numerOfRows); bitSet.flip(0, numerOfRows); int startIndex = 0; - byte[][] filterValues = dimColumnExecuterInfo.getExcludeFilterKeys(); for (int i = 0; i < filterValues.length; i++) { if (startIndex >= numerOfRows) { break; @@ -381,7 +388,6 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter { int numerOfRows) { BitSet bitSet = new BitSet(numerOfRows); bitSet.flip(0, numerOfRows); - byte[][] filterValues = dimColumnExecuterInfo.getExcludeFilterKeys(); // filterValues can be null when the dictionary chunk and surrogate size both are one if (filterValues.length == 0) { return bitSet; @@ -407,7 +413,7 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter { if (filterValues.length > 1) { for (int i = 0; i < numerOfRows; i++) { int index = CarbonUtil.binarySearch(filterValues, 0, filterValues.length - 1, - dimensionColumnPage.getChunkData(i)); + dimensionColumnPage, i); if (index >= 0) { bitSet.flip(i); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/4935cb18/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java index 516ed41..96ff7af 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java @@ -53,6 +53,13 @@ public class IncludeFilterExecuterImpl implements FilterExecuter { */ private boolean isNaturalSorted = false; + private byte[][] filterValues; + + public IncludeFilterExecuterImpl(byte[][] filterValues, boolean isNaturalSorted) { + this.filterValues = filterValues; + this.isNaturalSorted = isNaturalSorted; + } + public IncludeFilterExecuterImpl(DimColumnResolvedFilterInfo dimColumnEvaluatorInfo, MeasureColumnResolvedFilterInfo msrColumnEvaluatorInfo, SegmentProperties segmentProperties, boolean isMeasure) { @@ -103,7 +110,13 @@ public class IncludeFilterExecuterImpl implements FilterExecuter { if (dimensionRawColumnChunk.getMaxValues() != null) { if (isScanRequired(dimensionRawColumnChunk.getMaxValues()[i], dimensionRawColumnChunk.getMinValues()[i], dimColumnExecuterInfo.getFilterKeys())) { - BitSet bitSet = getFilteredIndexes(dimensionRawColumnChunk.decodeColumnPage(i), + DimensionColumnPage dimensionColumnPage = dimensionRawColumnChunk.decodeColumnPage(i); + if (null == filterValues) { + filterValues = FilterUtil + .getEncodedFilterValues(dimensionRawColumnChunk.getLocalDictionary(), + dimColumnExecuterInfo.getFilterKeys()); + } + BitSet bitSet = getFilteredIndexes(dimensionColumnPage, dimensionRawColumnChunk.getRowCount()[i], useBitsetPipeLine, rawBlockletColumnChunks.getBitSetGroup(), i); bitSetGroup.setBitSet(bitSet, i); @@ -302,11 +315,11 @@ public class IncludeFilterExecuterImpl implements FilterExecuter { * @param pageNumber * @return filtered indexes bitset */ - private BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage, + protected BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage, int numberOfRows, boolean useBitsetPipeLine, BitSetGroup prvBitSetGroup, int pageNumber) { // check whether previous indexes can be optimal to apply filter on dimension column if (CarbonUtil.usePreviousFilterBitsetGroup(useBitsetPipeLine, prvBitSetGroup, pageNumber, - dimColumnExecuterInfo.getFilterKeys().length)) { + filterValues.length)) { return getFilteredIndexesUisngPrvBitset(dimensionColumnPage, prvBitSetGroup, pageNumber, numberOfRows); } else { @@ -338,7 +351,6 @@ public class IncludeFilterExecuterImpl implements FilterExecuter { return prvPageBitSet; } BitSet bitSet = new BitSet(numberOfRows); - byte[][] filterKeys = dimColumnExecuterInfo.getFilterKeys(); int compareResult = 0; // if dimension data was natural sorted then get the index from previous bitset // and use the same in next column data, otherwise use the inverted index reverse @@ -346,7 +358,7 @@ public class IncludeFilterExecuterImpl implements FilterExecuter { for (int index = prvPageBitSet.nextSetBit(0); index >= 0; index = prvPageBitSet.nextSetBit(index + 1)) { compareResult = CarbonUtil - .isFilterPresent(filterKeys, dimensionColumnPage, 0, filterKeys.length - 1, index); + .isFilterPresent(filterValues, dimensionColumnPage, 0, filterValues.length - 1, index); if (compareResult == 0) { bitSet.set(index); } @@ -355,7 +367,7 @@ public class IncludeFilterExecuterImpl implements FilterExecuter { for (int index = prvPageBitSet.nextSetBit(0); index >= 0; index = prvPageBitSet.nextSetBit(index + 1)) { compareResult = CarbonUtil - .isFilterPresent(filterKeys, dimensionColumnPage, 0, filterKeys.length - 1, + .isFilterPresent(filterValues, dimensionColumnPage, 0, filterValues.length - 1, dimensionColumnPage.getInvertedReverseIndex(index)); if (compareResult == 0) { bitSet.set(index); @@ -368,7 +380,6 @@ public class IncludeFilterExecuterImpl implements FilterExecuter { DimensionColumnPage dimensionColumnPage, int numerOfRows) { BitSet bitSet = new BitSet(numerOfRows); int startIndex = 0; - byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys(); for (int i = 0; i < filterValues.length; i++) { if (startIndex >= numerOfRows) { break; @@ -389,7 +400,6 @@ public class IncludeFilterExecuterImpl implements FilterExecuter { private BitSet setFilterdIndexToBitSet(DimensionColumnPage dimensionColumnPage, int numerOfRows) { BitSet bitSet = new BitSet(numerOfRows); - byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys(); // binary search can only be applied if column is sorted and // inverted index exists for that column if (isNaturalSorted) { @@ -412,7 +422,7 @@ public class IncludeFilterExecuterImpl implements FilterExecuter { if (filterValues.length > 1) { for (int i = 0; i < numerOfRows; i++) { int index = CarbonUtil.binarySearch(filterValues, 0, filterValues.length - 1, - dimensionColumnPage.getChunkData(i)); + dimensionColumnPage, i); if (index >= 0) { bitSet.set(i); } @@ -431,7 +441,7 @@ public class IncludeFilterExecuterImpl implements FilterExecuter { @Override public BitSet isScanRequired(byte[][] blkMaxVal, byte[][] blkMinVal) { BitSet bitSet = new BitSet(1); - byte[][] filterValues = null; + byte[][] filterValues; int columnIndex = 0; int chunkIndex = 0; boolean isScanRequired = false; http://git-wip-us.apache.org/repos/asf/carbondata/blob/4935cb18/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java index b218813..a4bbdd0 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java @@ -72,6 +72,7 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl { private boolean startBlockMinIsDefaultStart; private boolean endBlockMaxisDefaultEnd; private boolean isRangeFullyCoverBlock; + private boolean isNaturalSorted; public RangeValueFilterExecuterImpl(DimColumnResolvedFilterInfo dimColEvaluatorInfo, Expression exp, byte[][] filterRangeValues, SegmentProperties segmentProperties) { @@ -89,7 +90,10 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl { isRangeFullyCoverBlock = false; initDimensionChunkIndexes(); ifDefaultValueMatchesFilter(); - + if (isDimensionPresentInCurrentBlock == true) { + isNaturalSorted = dimColEvaluatorInfo.getDimension().isUseInvertedIndex() + && dimColEvaluatorInfo.getDimension().isSortColumn(); + } } /** @@ -139,7 +143,7 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl { */ public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks, boolean useBitsetPipeLine) throws FilterUnsupportedException, IOException { - return applyNoAndDirectFilter(rawBlockletColumnChunks); + return applyNoAndDirectFilter(rawBlockletColumnChunks, useBitsetPipeLine); } /** @@ -328,8 +332,8 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl { * @return * @throws IOException */ - private BitSetGroup applyNoAndDirectFilter(RawBlockletColumnChunks blockChunkHolder) - throws IOException { + private BitSetGroup applyNoAndDirectFilter(RawBlockletColumnChunks blockChunkHolder, + boolean useBitsetPipeLine) throws IOException { // In case of Alter Table Add and Delete Columns the isDimensionPresentInCurrentBlock can be // false, in that scenario the default values of the column should be shown. @@ -352,6 +356,8 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl { DimensionRawColumnChunk rawColumnChunk = blockChunkHolder.getDimensionRawColumnChunks()[chunkIndex]; BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount()); + FilterExecuter filterExecuter = null; + boolean isExclude = false; for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) { if (rawColumnChunk.getMaxValues() != null) { if (isScanRequired(rawColumnChunk.getMinValues()[i], rawColumnChunk.getMaxValues()[i], @@ -362,8 +368,31 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl { bitSet.flip(0, rawColumnChunk.getRowCount()[i]); bitSetGroup.setBitSet(bitSet, i); } else { - BitSet bitSet = getFilteredIndexes(rawColumnChunk.decodeColumnPage(i), - rawColumnChunk.getRowCount()[i]); + BitSet bitSet; + DimensionColumnPage dimensionColumnPage = rawColumnChunk.decodeColumnPage(i); + if (null != rawColumnChunk.getLocalDictionary()) { + if (null == filterExecuter) { + filterExecuter = FilterUtil + .getFilterExecutorForLocalDictionary(rawColumnChunk, exp, isNaturalSorted); + if (filterExecuter instanceof ExcludeFilterExecuterImpl) { + isExclude = true; + } + } + if (!isExclude) { + bitSet = ((IncludeFilterExecuterImpl) filterExecuter) + .getFilteredIndexes(dimensionColumnPage, + rawColumnChunk.getRowCount()[i], useBitsetPipeLine, + blockChunkHolder.getBitSetGroup(), i); + } else { + bitSet = ((ExcludeFilterExecuterImpl) filterExecuter) + .getFilteredIndexes(dimensionColumnPage, + rawColumnChunk.getRowCount()[i], useBitsetPipeLine, + blockChunkHolder.getBitSetGroup(), i); + } + } else { + bitSet = getFilteredIndexes(dimensionColumnPage, + rawColumnChunk.getRowCount()[i]); + } bitSetGroup.setBitSet(bitSet, i); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/4935cb18/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java index e339a58..f2cb3dd 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java @@ -194,6 +194,8 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute DimensionRawColumnChunk rawColumnChunk = rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]; BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount()); + FilterExecuter filterExecuter = null; + boolean isExclude = false; for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) { if (rawColumnChunk.getMaxValues() != null) { if (isScanRequired(rawColumnChunk.getMaxValues()[i], this.filterRangeValues)) { @@ -204,8 +206,31 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute bitSet.flip(0, rawColumnChunk.getRowCount()[i]); bitSetGroup.setBitSet(bitSet, i); } else { - BitSet bitSet = getFilteredIndexes(rawColumnChunk.decodeColumnPage(i), - rawColumnChunk.getRowCount()[i]); + BitSet bitSet = null; + DimensionColumnPage dimensionColumnPage = rawColumnChunk.decodeColumnPage(i); + if (null != rawColumnChunk.getLocalDictionary()) { + if (null == filterExecuter) { + filterExecuter = FilterUtil + .getFilterExecutorForLocalDictionary(rawColumnChunk, exp, isNaturalSorted); + if (filterExecuter instanceof ExcludeFilterExecuterImpl) { + isExclude = true; + } + } + if (!isExclude) { + bitSet = ((IncludeFilterExecuterImpl) filterExecuter) + .getFilteredIndexes(dimensionColumnPage, + rawColumnChunk.getRowCount()[i], useBitsetPipeLine, + rawBlockletColumnChunks.getBitSetGroup(), i); + } else { + bitSet = ((ExcludeFilterExecuterImpl) filterExecuter) + .getFilteredIndexes(dimensionColumnPage, + rawColumnChunk.getRowCount()[i], useBitsetPipeLine, + rawBlockletColumnChunks.getBitSetGroup(), i); + } + } else { + bitSet = getFilteredIndexes(dimensionColumnPage, + rawColumnChunk.getRowCount()[i]); + } bitSetGroup.setBitSet(bitSet, i); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/4935cb18/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java index 2ea3c73..b893f99 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java @@ -191,6 +191,8 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte DimensionRawColumnChunk rawColumnChunk = rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]; BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount()); + FilterExecuter filterExecuter = null; + boolean isExclude = false; for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) { if (rawColumnChunk.getMaxValues() != null) { if (isScanRequired(rawColumnChunk.getMaxValues()[i], this.filterRangeValues)) { @@ -201,8 +203,31 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte bitSet.flip(0, rawColumnChunk.getRowCount()[i]); bitSetGroup.setBitSet(bitSet, i); } else { - BitSet bitSet = getFilteredIndexes(rawColumnChunk.decodeColumnPage(i), - rawColumnChunk.getRowCount()[i]); + DimensionColumnPage dimensionColumnPage = rawColumnChunk.decodeColumnPage(i); + BitSet bitSet = null; + if (null != rawColumnChunk.getLocalDictionary()) { + if (null == filterExecuter) { + filterExecuter = FilterUtil + .getFilterExecutorForLocalDictionary(rawColumnChunk, exp, isNaturalSorted); + if (filterExecuter instanceof ExcludeFilterExecuterImpl) { + isExclude = true; + } + } + if (!isExclude) { + bitSet = ((IncludeFilterExecuterImpl) filterExecuter) + .getFilteredIndexes(dimensionColumnPage, + rawColumnChunk.getRowCount()[i], useBitsetPipeLine, + rawBlockletColumnChunks.getBitSetGroup(), i); + } else { + bitSet = ((ExcludeFilterExecuterImpl) filterExecuter) + .getFilteredIndexes(dimensionColumnPage, + rawColumnChunk.getRowCount()[i], useBitsetPipeLine, + rawBlockletColumnChunks.getBitSetGroup(), i); + } + } else { + bitSet = getFilteredIndexes(dimensionColumnPage, + rawColumnChunk.getRowCount()[i]); + } bitSetGroup.setBitSet(bitSet, i); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/4935cb18/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java index f52d087..1b0a64a 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java @@ -193,11 +193,36 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter DimensionRawColumnChunk rawColumnChunk = rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]; BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount()); + FilterExecuter filterExecuter = null; + boolean isExclude = false; for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) { if (rawColumnChunk.getMinValues() != null) { if (isScanRequired(rawColumnChunk.getMinValues()[i], this.filterRangeValues)) { - BitSet bitSet = getFilteredIndexes(rawColumnChunk.decodeColumnPage(i), - rawColumnChunk.getRowCount()[i]); + BitSet bitSet; + DimensionColumnPage dimensionColumnPage = rawColumnChunk.decodeColumnPage(i); + if (null != rawColumnChunk.getLocalDictionary()) { + if (null == filterExecuter) { + filterExecuter = FilterUtil + .getFilterExecutorForLocalDictionary(rawColumnChunk, exp, isNaturalSorted); + if (filterExecuter instanceof ExcludeFilterExecuterImpl) { + isExclude = true; + } + } + if (!isExclude) { + bitSet = ((IncludeFilterExecuterImpl) filterExecuter) + .getFilteredIndexes(dimensionColumnPage, + rawColumnChunk.getRowCount()[i], useBitsetPipeLine, + rawBlockletColumnChunks.getBitSetGroup(), i); + } else { + bitSet = ((ExcludeFilterExecuterImpl) filterExecuter) + .getFilteredIndexes(dimensionColumnPage, + rawColumnChunk.getRowCount()[i], useBitsetPipeLine, + rawBlockletColumnChunks.getBitSetGroup(), i); + } + } else { + bitSet = getFilteredIndexes(dimensionColumnPage, + rawColumnChunk.getRowCount()[i]); + } bitSetGroup.setBitSet(bitSet, i); } } else { http://git-wip-us.apache.org/repos/asf/carbondata/blob/4935cb18/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFilterExecuterImpl.java index 97e750a..5f68e34 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFilterExecuterImpl.java @@ -193,11 +193,36 @@ public class RowLevelRangeLessThanFilterExecuterImpl extends RowLevelFilterExecu DimensionRawColumnChunk rawColumnChunk = rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]; BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount()); + FilterExecuter filterExecuter = null; + boolean isExclude = false; for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) { if (rawColumnChunk.getMinValues() != null) { if (isScanRequired(rawColumnChunk.getMinValues()[i], this.filterRangeValues)) { - BitSet bitSet = getFilteredIndexes(rawColumnChunk.decodeColumnPage(i), - rawColumnChunk.getRowCount()[i]); + BitSet bitSet; + DimensionColumnPage dimensionColumnPage = rawColumnChunk.decodeColumnPage(i); + if (null != rawColumnChunk.getLocalDictionary()) { + if (null == filterExecuter) { + filterExecuter = FilterUtil + .getFilterExecutorForLocalDictionary(rawColumnChunk, exp, isNaturalSorted); + if (filterExecuter instanceof ExcludeFilterExecuterImpl) { + isExclude = true; + } + } + if (!isExclude) { + bitSet = ((IncludeFilterExecuterImpl) filterExecuter) + .getFilteredIndexes(dimensionColumnPage, + rawColumnChunk.getRowCount()[i], useBitsetPipeLine, + rawBlockletColumnChunks.getBitSetGroup(), i); + } else { + bitSet = ((ExcludeFilterExecuterImpl) filterExecuter) + .getFilteredIndexes(dimensionColumnPage, + rawColumnChunk.getRowCount()[i], useBitsetPipeLine, + rawBlockletColumnChunks.getBitSetGroup(), i); + } + } else { + bitSet = getFilteredIndexes(dimensionColumnPage, + rawColumnChunk.getRowCount()[i]); + } bitSetGroup.setBitSet(bitSet, i); } } else { http://git-wip-us.apache.org/repos/asf/carbondata/blob/4935cb18/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java index c568f90..789897e 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java @@ -505,25 +505,19 @@ public final class CarbonUtil { * @param filterValues * @param low * @param high - * @param compareValue * @return the compareValue's index in the filterValues */ public static int binarySearch(byte[][] filterValues, int low, int high, - byte[] compareValue) { - + DimensionColumnPage dimensionColumnPage, int rowId) { rangeCheck(low, high); - while (low <= high) { int mid = (low + high) >>> 1; - - int result = ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[mid], compareValue); - + int result = dimensionColumnPage.compareTo(rowId, filterValues[mid]); if (result < 0) { - low = mid + 1; - } else if (result > 0) { high = mid - 1; + } else if (result > 0) { + low = mid + 1; } else { - return mid; // key found } http://git-wip-us.apache.org/repos/asf/carbondata/blob/4935cb18/core/src/test/java/org/apache/carbondata/core/localdictionary/TestPageLevelDictionary.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/localdictionary/TestPageLevelDictionary.java b/core/src/test/java/org/apache/carbondata/core/localdictionary/TestPageLevelDictionary.java index da62133..a2a25bd 100644 --- a/core/src/test/java/org/apache/carbondata/core/localdictionary/TestPageLevelDictionary.java +++ b/core/src/test/java/org/apache/carbondata/core/localdictionary/TestPageLevelDictionary.java @@ -29,7 +29,7 @@ public class TestPageLevelDictionary { LocalDictionaryGenerator generator = new ColumnLocalDictionaryGenerator(1000, 2); String columnName = "column1"; PageLevelDictionary pageLevelDictionary = new PageLevelDictionary(generator, columnName, - DataTypes.STRING); + DataTypes.STRING, false); try { for (int i = 1; i <= 1000; i++) { Assert.assertTrue((i + 1) == pageLevelDictionary.getDictionaryValue(("" + i).getBytes())); @@ -43,7 +43,7 @@ public class TestPageLevelDictionary { @Test public void testPageLevelDictionaryContainsOnlyUsedDictionaryValues() { LocalDictionaryGenerator generator = new ColumnLocalDictionaryGenerator(1000, 2); String columnName = "column1"; - PageLevelDictionary pageLevelDictionary1 = new PageLevelDictionary(generator, columnName, DataTypes.STRING); + PageLevelDictionary pageLevelDictionary1 = new PageLevelDictionary(generator, columnName, DataTypes.STRING, false); byte[][] validateData = new byte[500][]; try { for (int i = 1; i <= 500; i++) { @@ -58,7 +58,7 @@ public class TestPageLevelDictionary { } catch (DictionaryThresholdReachedException e) { Assert.assertTrue(false); } - PageLevelDictionary pageLevelDictionary2 = new PageLevelDictionary(generator, columnName, DataTypes.STRING); + PageLevelDictionary pageLevelDictionary2 = new PageLevelDictionary(generator, columnName, DataTypes.STRING, false); try { for (int i = 1; i <= 500; i++) { byte[] data = ("vikas" + i).getBytes(); @@ -95,7 +95,7 @@ public class TestPageLevelDictionary { public void testPageLevelDictionaryContainsOnlyUsedDictionaryValuesWhenMultiplePagesUseSameDictionary() { LocalDictionaryGenerator generator = new ColumnLocalDictionaryGenerator(1000, 2); String columnName = "column1"; - PageLevelDictionary pageLevelDictionary1 = new PageLevelDictionary(generator, columnName, DataTypes.STRING); + PageLevelDictionary pageLevelDictionary1 = new PageLevelDictionary(generator, columnName, DataTypes.STRING, false); byte[][] validateData = new byte[10][]; int index = 0; try { @@ -112,7 +112,7 @@ public class TestPageLevelDictionary { } catch (DictionaryThresholdReachedException e) { Assert.assertTrue(false); } - PageLevelDictionary pageLevelDictionary2 = new PageLevelDictionary(generator, columnName, DataTypes.STRING); + PageLevelDictionary pageLevelDictionary2 = new PageLevelDictionary(generator, columnName, DataTypes.STRING, false); try { for (int i = 1; i <= 5; i++) { byte[] data = ("vikas" + i).getBytes(); http://git-wip-us.apache.org/repos/asf/carbondata/blob/4935cb18/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImplTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImplTest.java b/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImplTest.java index 11c4980..00d71a7 100644 --- a/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImplTest.java +++ b/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImplTest.java @@ -31,7 +31,7 @@ public class ExcludeFilterExecuterImplTest extends IncludeFilterExecuterImplTest if (filterValues.length > 1) { for (int j = 0; j < numerOfRows; j++) { int index = CarbonUtil.binarySearch(filterValues, 0, filterValues.length - 1, - dimColumnDataChunk.getChunkData(j)); + dimColumnDataChunk, j); if (index >= 0) { bitSet.flip(j); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/4935cb18/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImplTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImplTest.java b/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImplTest.java index 1bfa875..92c2e85 100644 --- a/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImplTest.java +++ b/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImplTest.java @@ -46,7 +46,7 @@ public class IncludeFilterExecuterImplTest extends TestCase { if (filterValues.length > 1) { for (int i = 0; i < numerOfRows; i++) { int index = CarbonUtil.binarySearch(filterValues, 0, filterValues.length - 1, - dimensionColumnPage.getChunkData(i)); + dimensionColumnPage, i); if (index >= 0) { bitSet.set(i); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/4935cb18/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java index 4846841..d75f323 100644 --- a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java +++ b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java @@ -1098,8 +1098,8 @@ public class CarbonUtilTest { assertEquals(expectRangeIndex[0], range[0]); assertEquals(expectRangeIndex[1], range[1]); - int index = CarbonUtil.binarySearch(dataArr, 0, dataChunk.length / keyWord.length - 1, keyWord); - assertTrue(expectRangeIndex[0] <= index && index <= range[1]); +// int index = CarbonUtil.binarySearch(dataArr, 0, dataChunk.length / keyWord.length - 1, keyWord); +// assertTrue(expectRangeIndex[0] <= index && index <= range[1]); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/4935cb18/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/TableLevelCompactionOptionTest.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/TableLevelCompactionOptionTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/TableLevelCompactionOptionTest.scala index 5445779..458d656 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/TableLevelCompactionOptionTest.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/TableLevelCompactionOptionTest.scala @@ -132,7 +132,7 @@ class TableLevelCompactionOptionTest extends QueryTest |(id INT, name STRING, city STRING, age INT) |STORED BY 'org.apache.carbondata.format' |TBLPROPERTIES('SORT_COLUMNS'='city,name', - |'MAJOR_COMPACTION_SIZE'='1') + |'MAJOR_COMPACTION_SIZE'='1', 'LOCAL_DICTIONARY_ENABLE'='false') """.stripMargin) for (i <- 0 until 2) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/4935cb18/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala index e61d99c..bdbd43b 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala @@ -207,7 +207,7 @@ object DataMapWriterSuite { pages: Array[ColumnPage]): Unit = { assert(pages.length == 1) assert(pages(0).getDataType == DataTypes.STRING) - val bytes: Array[Byte] = pages(0).getByteArrayPage()(0) + val bytes: Array[Byte] = pages(0).getBytes(0) assert(bytes.sameElements(Seq(0, 1, 'b'.toByte))) callbackSeq :+= s"add page data: blocklet $blockletId, page $pageId" } http://git-wip-us.apache.org/repos/asf/carbondata/blob/4935cb18/integration/spark2/src/test/scala/org/apache/spark/sql/CarbonGetTableDetailComandTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/test/scala/org/apache/spark/sql/CarbonGetTableDetailComandTestCase.scala b/integration/spark2/src/test/scala/org/apache/spark/sql/CarbonGetTableDetailComandTestCase.scala index 1c7cb10..c6c647c 100644 --- a/integration/spark2/src/test/scala/org/apache/spark/sql/CarbonGetTableDetailComandTestCase.scala +++ b/integration/spark2/src/test/scala/org/apache/spark/sql/CarbonGetTableDetailComandTestCase.scala @@ -25,11 +25,11 @@ import org.scalatest.BeforeAndAfterAll class CarbonGetTableDetailCommandTestCase extends QueryTest with BeforeAndAfterAll { override def beforeAll(): Unit = { sql("drop table if exists table_info1") - sql("create table table_info1 (empno int, workgroupcategory string, deptno int, projectcode int, attendance int) stored by 'org.apache.carbondata.format'") + sql("create table table_info1 (empno int, workgroupcategory string, deptno int, projectcode int, attendance int) stored by 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false')") sql(s"""load data local inpath '$resourcesPath/data.csv' into table table_info1 options('delimiter'=',', 'quotechar'='\"', 'fileheader'='')""") sql("drop table if exists table_info2") - sql("create table table_info2 (empno int, workgroupcategory string, deptno int, projectcode int, attendance int) stored by 'org.apache.carbondata.format'") + sql("create table table_info2 (empno int, workgroupcategory string, deptno int, projectcode int, attendance int) stored by 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false')") sql(s"""load data local inpath '$resourcesPath/data.csv' into table table_info2 options('delimiter'=',', 'quotechar'='\"', 'fileheader'='')""") }
