Github user ravipesala commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2447#discussion_r200374220
--- Diff:
core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java ---
@@ -1839,4 +1844,118 @@ public static void
removeInExpressionNodeWithPositionIdColumn(Expression express
}
}
}
+
+ public static byte[][] getEncodedFilterValues(CarbonDictionary
dictionary,
+ byte[][] actualFilterValues) {
+ if (null == dictionary) {
+ return actualFilterValues;
+ }
+ KeyGenerator keyGenerator = KeyGeneratorFactory.getKeyGenerator(new
int[] { 100000 });
+ List<byte[]> encodedFilters = new ArrayList<>();
+ for (byte[] actualFilter : actualFilterValues) {
+ for (int i = 1; i < dictionary.getDictionaryValues().length; i++) {
+ if (ByteUtil.UnsafeComparer.INSTANCE
+ .compareTo(actualFilter, dictionary.getDictionaryValues()[i])
+ == 0) {
+ try {
+ encodedFilters.add(keyGenerator.generateKey(new int[] { i }));
+ } catch (KeyGenException e) {
+ //do nothing
+ }
+ break;
+ }
+ }
+ }
+ return getSortedEncodedFilters(encodedFilters);
+ }
+
+ private static byte[][] getSortedEncodedFilters(List<byte[]>
encodedFilters) {
+ java.util.Comparator<byte[]> filterNoDictValueComaparator = new
java.util.Comparator<byte[]>() {
+ @Override public int compare(byte[] filterMember1, byte[]
filterMember2) {
+ return ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterMember1,
filterMember2);
+ }
+ };
+ Collections.sort(encodedFilters, filterNoDictValueComaparator);
+ return encodedFilters.toArray(new byte[encodedFilters.size()][]);
+ }
+
+ private static BitSet getIncludeDictionaryValues(Expression expression,
+ CarbonDictionary dictionary) throws FilterUnsupportedException {
+ ConditionalExpression conExp = (ConditionalExpression) expression;
+ ColumnExpression columnExpression = conExp.getColumnList().get(0);
+ BitSet includeFilterBitSet = new BitSet();
+ for (int i = 2; i < dictionary.getDictionaryValues().length; i++) {
+ try {
+ RowIntf row = new RowImpl();
+ String stringValue = new
String(dictionary.getDictionaryValues()[i],
+ Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
+ row.setValues(new Object[] {
DataTypeUtil.getDataBasedOnDataType(stringValue,
+ columnExpression.getCarbonColumn().getDataType()) });
+ Boolean rslt = expression.evaluate(row).getBoolean();
+ if (null != rslt) {
+ if (rslt) {
+ includeFilterBitSet.set(i);
+ }
+ }
+ } catch (FilterIllegalMemberException e) {
+ LOGGER.debug(e.getMessage());
+ }
+ }
+ return includeFilterBitSet;
+ }
+
+ public static byte[][] getEncodedFilterValues(BitSet includeDictValues,
int dictSize,
+ boolean useExclude) {
+ KeyGenerator keyGenerator = KeyGeneratorFactory
+ .getKeyGenerator(new int[] {
CarbonCommonConstants.LOCAL_DICTIONARY_MAX });
+ List<byte[]> encodedFilterValues = new ArrayList<>();
+ int[] dummy = new int[1];
+ if (!useExclude) {
+ try {
+ for (int i = includeDictValues.nextSetBit(0);
+ i >= 0; i = includeDictValues.nextSetBit(i + 1)) {
+ dummy[0] = i;
+ encodedFilterValues.add(keyGenerator.generateKey(dummy));
+ }
+ } catch (KeyGenException e) {
+ // do nothing
+ }
+ return encodedFilterValues.toArray(new
byte[encodedFilterValues.size()][]);
+ } else {
+ try {
+ for (int i = 1; i < dictSize; i++) {
+ if (!includeDictValues.get(i)) {
+ dummy[0] = i;
+ encodedFilterValues.add(keyGenerator.generateKey(dummy));
+ }
+ }
+ } catch (KeyGenException e) {
+ // do nothing
+ }
+ }
+ return getSortedEncodedFilters(encodedFilterValues);
+ }
+
+ public static FilterExecuter getFilterExecutorForLocalDictionary(
--- End diff --
Change name appropriately as it is only for range filters
---
