http://git-wip-us.apache.org/repos/asf/carbondata/blob/4e835095/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeTypeExecuterFacory.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeTypeExecuterFacory.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeTypeExecuterFacory.java index 53cbc66..3469a54 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeTypeExecuterFacory.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeTypeExecuterFacory.java @@ -49,7 +49,9 @@ public class RowLevelRangeTypeExecuterFacory { filterExpressionResolverTree.getFilterExpression(), ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree).getTableIdentifier(), ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree) - .getFilterRangeValues(segmentProperties), segmentProperties); + .getFilterRangeValues(segmentProperties), + ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree) + .getMeasureFilterRangeValues(), segmentProperties); case ROWLEVEL_LESSTHAN_EQUALTO: return new RowLevelRangeLessThanEqualFilterExecuterImpl( ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree) @@ -59,7 +61,9 @@ public class RowLevelRangeTypeExecuterFacory { filterExpressionResolverTree.getFilterExpression(), ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree).getTableIdentifier(), ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree) - .getFilterRangeValues(segmentProperties), segmentProperties); + .getFilterRangeValues(segmentProperties), + ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree) + .getMeasureFilterRangeValues(), segmentProperties); case ROWLEVEL_GREATERTHAN_EQUALTO: return new RowLevelRangeGrtrThanEquaToFilterExecuterImpl( ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree) @@ -69,7 +73,9 @@ public class RowLevelRangeTypeExecuterFacory { filterExpressionResolverTree.getFilterExpression(), ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree).getTableIdentifier(), ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree) - .getFilterRangeValues(segmentProperties), segmentProperties); + .getFilterRangeValues(segmentProperties), + ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree) + .getMeasureFilterRangeValues(), segmentProperties); case ROWLEVEL_GREATERTHAN: return new RowLevelRangeGrtThanFiterExecuterImpl( ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree) @@ -79,7 +85,9 @@ public class RowLevelRangeTypeExecuterFacory { filterExpressionResolverTree.getFilterExpression(), ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree).getTableIdentifier(), ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree) - .getFilterRangeValues(segmentProperties), segmentProperties); + .getFilterRangeValues(segmentProperties), + ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree) + .getMeasureFilterRangeValues(), segmentProperties); default: // Scenario wont come logic must break return null;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4e835095/core/src/main/java/org/apache/carbondata/core/scan/filter/partition/PartitionFilterUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/partition/PartitionFilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/partition/PartitionFilterUtil.java index a056d94..d040c1b 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/partition/PartitionFilterUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/partition/PartitionFilterUtil.java @@ -30,8 +30,6 @@ import org.apache.carbondata.core.util.ByteUtil; import org.apache.carbondata.core.util.comparator.Comparator; import org.apache.carbondata.core.util.comparator.SerializableComparator; -import org.apache.spark.sql.types.Decimal; - public class PartitionFilterUtil { /** http://git-wip-us.apache.org/repos/asf/carbondata/blob/4e835095/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java index 95d3223..8d3c366 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java @@ -317,7 +317,7 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf { } else if (null != dimColResolvedFilterInfo.getFilterValues() && dimColResolvedFilterInfo .getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) { return FilterUtil.getKeyArray(this.dimColResolvedFilterInfo.getFilterValues(), - this.dimColResolvedFilterInfo.getDimension(), null, segmentProperties); + this.dimColResolvedFilterInfo.getDimension(), segmentProperties); } return null; http://git-wip-us.apache.org/repos/asf/carbondata/blob/4e835095/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java index 3bdca36..f64342f 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java @@ -87,18 +87,23 @@ public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverIm .getDimensionFromCurrentBlock(this.dimColEvaluatorInfoList.get(0).getDimension()); if (null != dimensionFromCurrentBlock) { return FilterUtil.getKeyArray(this.dimColEvaluatorInfoList.get(0).getFilterValues(), - dimensionFromCurrentBlock, null, segmentProperties); + dimensionFromCurrentBlock, segmentProperties); } - } else if (msrColEvalutorInfoList.size() > 0 && null != msrColEvalutorInfoList.get(0) + } + return null; + } + + public Object[] getMeasureFilterRangeValues() { + if (msrColEvalutorInfoList.size() > 0 && null != msrColEvalutorInfoList.get(0) .getFilterValues()) { - List<byte[]> measureFilterValuesList = + List<Object> measureFilterValuesList = msrColEvalutorInfoList.get(0).getFilterValues().getMeasuresFilterValuesList(); - return measureFilterValuesList.toArray((new byte[measureFilterValuesList.size()][])); + return measureFilterValuesList.toArray((new Object[measureFilterValuesList.size()])); } return null; - } + /** * method will get the start key based on the filter surrogates * @@ -179,23 +184,22 @@ public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverIm return filterValuesList; } - private List<byte[]> getMeasureRangeValues(CarbonMeasure carbonMeasure) { + private List<Object> getMeasureRangeValues(CarbonMeasure carbonMeasure) { List<ExpressionResult> listOfExpressionResults = new ArrayList<ExpressionResult>(20); if (this.getFilterExpression() instanceof BinaryConditionalExpression) { listOfExpressionResults = ((BinaryConditionalExpression) this.getFilterExpression()).getLiterals(); } - List<byte[]> filterValuesList = new ArrayList<byte[]>(20); + List<Object> filterValuesList = new ArrayList<>(20); boolean invalidRowsPresent = false; for (ExpressionResult result : listOfExpressionResults) { try { if (result.getString() == null) { - filterValuesList.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL.getBytes()); + filterValuesList.add(null); continue; } - filterValuesList.add(DataTypeUtil - .getMeasureByteArrayBasedOnDataTypes(result.getString(), - result.getDataType(), carbonMeasure)); + filterValuesList.add(DataTypeUtil.getMeasureValueBasedOnDataType(result.getString(), + result.getDataType(), carbonMeasure)); } catch (FilterIllegalMemberException e) { // Any invalid member while evaluation shall be ignored, system will log the // error only once since all rows the evaluation happens so inorder to avoid @@ -203,13 +207,8 @@ public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverIm FilterUtil.logError(e, invalidRowsPresent); } } - Comparator<byte[]> filterMeasureComaparator = new Comparator<byte[]>() { - @Override public int compare(byte[] filterMember1, byte[] filterMember2) { - return ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterMember1, filterMember2); - } - - }; - Collections.sort(filterValuesList, filterMeasureComaparator); + Collections.sort(filterValuesList, org.apache.carbondata.core.util.comparator.Comparator + .getComparatorByDataTypeForMeasure(carbonMeasure.getDataType())); return filterValuesList; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/4e835095/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java index 60546ed..db5010f 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java @@ -22,7 +22,6 @@ import java.io.ObjectOutputStream; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.Set; import org.apache.carbondata.common.logging.LogService; @@ -440,29 +439,6 @@ public class CarbonMetadataUtil { return false; } - private static ByteBuffer writeInfoIfDecimal(int blockIndex, - SegmentProperties segmentProperties) { - Map<Integer, Integer> blockMapping = segmentProperties.getMeasuresOrdinalToBlockMapping(); - List<CarbonMeasure> measures = segmentProperties.getMeasures(); - CarbonMeasure selectedMeasure = null; - for (CarbonMeasure measure : measures) { - Integer blockId = blockMapping.get(measure.getOrdinal()); - selectedMeasure = measure; - if (blockId == blockIndex) { - break; - } - } - assert (selectedMeasure != null); - if (selectedMeasure.getDataType() == DataType.DECIMAL) { - ByteBuffer buffer = ByteBuffer.allocate(8); - buffer.putInt(selectedMeasure.getScale()); - buffer.putInt(selectedMeasure.getPrecision()); - buffer.flip(); - return buffer; - } - return null; - } - private static byte[] serializeEncoderMeta(ValueEncoderMeta encoderMeta) throws IOException { // TODO : should remove the unnecessary fields. ByteArrayOutputStream aos = new ByteArrayOutputStream(); http://git-wip-us.apache.org/repos/asf/carbondata/blob/4e835095/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java index 5f146f6..6e54f1f 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java @@ -121,10 +121,15 @@ public final class DataTypeUtil { } public static Object getMeasureObjectFromDataType(byte[] data, DataType dataType) { + if (data == null || data.length == 0) { + return null; + } ByteBuffer bb = ByteBuffer.wrap(data); switch (dataType) { case SHORT: + return (short)bb.getLong(); case INT: + return (int)bb.getLong(); case LONG: return bb.getLong(); case DECIMAL: @@ -134,113 +139,13 @@ public final class DataTypeUtil { } } - /** - * This method will convert a given ByteArray to its specific type - * - * @param msrValue - * @param dataType - * @param carbonMeasure - * @return - */ - // public static byte[] getMeasureByteArrayBasedOnDataType(String msrValue, DataType dataType, - // CarbonMeasure carbonMeasure) { - // switch (dataType) { - // case DECIMAL: - // BigDecimal bigDecimal = - // new BigDecimal(msrValue).setScale(carbonMeasure.getScale(), RoundingMode.HALF_UP); - // return ByteUtil.toBytes(normalizeDecimalValue(bigDecimal, carbonMeasure.getPrecision())); - // case SHORT: - // return ByteUtil.toBytes((Short.parseShort(msrValue))); - // case INT: - // return ByteUtil.toBytes(Integer.parseInt(msrValue)); - // case LONG: - // return ByteUtil.toBytes(Long.valueOf(msrValue)); - // default: - // Double parsedValue = Double.valueOf(msrValue); - // if (Double.isInfinite(parsedValue) || Double.isNaN(parsedValue)) { - // return null; - // } - // return ByteUtil.toBytes(parsedValue); - // } - // } - public static byte[] getMeasureByteArrayBasedOnDataTypes(String msrValue, DataType dataType, - CarbonMeasure carbonMeasure) { - ByteBuffer b; - switch (dataType) { - case BYTE: - case SHORT: - case INT: - case LONG: - b = ByteBuffer.allocate(8); - b.putLong(Long.valueOf(msrValue)); - b.flip(); - return b.array(); - case DOUBLE: - b = ByteBuffer.allocate(8); - b.putDouble(Double.valueOf(msrValue)); - b.flip(); - return b.array(); - case DECIMAL: - BigDecimal bigDecimal = - new BigDecimal(msrValue).setScale(carbonMeasure.getScale(), RoundingMode.HALF_UP); - return DataTypeUtil - .bigDecimalToByte(normalizeDecimalValue(bigDecimal, carbonMeasure.getPrecision())); - default: - throw new IllegalArgumentException("Invalid data type: " + dataType); - } - } - - /** - * This method will convert a given ByteArray to its specific type - * - * @param msrValue - * @param dataType - * @param carbonMeasure - * @return - */ - public static byte[] getMeasureByteArrayBasedOnDataType(ColumnPage measurePage, int index, - DataType dataType, CarbonMeasure carbonMeasure) { - switch (dataType) { - case DECIMAL: - BigDecimal bigDecimal = new BigDecimal(measurePage.getDouble(index)) - .setScale(carbonMeasure.getScale(), RoundingMode.HALF_UP); - return ByteUtil.toBytes(normalizeDecimalValue(bigDecimal, carbonMeasure.getPrecision())); - case SHORT: - return ByteUtil.toBytes(measurePage.getShort(index)); - case INT: - return ByteUtil.toBytes(measurePage.getInt(index)); - case LONG: - return ByteUtil.toBytes(measurePage.getLong(index)); - default: - Double parsedValue = Double.valueOf(measurePage.getDouble(index)); - if (Double.isInfinite(parsedValue) || Double.isNaN(parsedValue)) { - return null; - } - return ByteUtil.toBytes(parsedValue); - } - } - public static Object getMeasureObjectBasedOnDataType(ColumnPage measurePage, int index, DataType dataType, CarbonMeasure carbonMeasure) { - // switch (dataType) { - // case DECIMAL: - // BigDecimal bigDecimal = new BigDecimal(measurePage.getDouble(index)) - // .setScale(carbonMeasure.getScale(), RoundingMode.HALF_UP); - // return normalizeDecimalValue(bigDecimal, carbonMeasure.getPrecision()); - // case SHORT: - // case INT: - // case LONG: - // return measurePage.getLong(index); - // default: - // Double parsedValue = Double.valueOf(measurePage.getDouble(index)); - // if (Double.isInfinite(parsedValue) || Double.isNaN(parsedValue)) { - // return null; - // } - // return parsedValue; - // } switch (dataType) { case SHORT: + return (short)measurePage.getLong(index); case INT: + return (int)measurePage.getLong(index); case LONG: return measurePage.getLong(index); case DECIMAL: @@ -249,7 +154,6 @@ public final class DataTypeUtil { bigDecimalMsrValue = bigDecimalMsrValue.setScale(carbonMeasure.getScale(), RoundingMode.HALF_UP); } - //return org.apache.spark.sql.types.Decimal.apply(bigDecimalMsrValue); return normalizeDecimalValue(bigDecimalMsrValue, carbonMeasure.getPrecision()); default: return measurePage.getDouble(index); @@ -379,6 +283,19 @@ public final class DataTypeUtil { * @return actual data after conversion */ public static Object getDataBasedOnDataType(String data, DataType actualDataType) { + return getDataBasedOnDataType(data, actualDataType, getDataTypeConverter()); + } + + /** + * Below method will be used to convert the data passed to its actual data + * type + * + * @param data data + * @param actualDataType actual data type + * @return actual data after conversion + */ + public static Object getDataBasedOnDataType(String data, DataType actualDataType, + DataTypeConverter converter) { if (null == data || CarbonCommonConstants.MEMBER_DEFAULT_VAL.equals(data)) { return null; } @@ -435,15 +352,14 @@ public final class DataTypeUtil { if (data.isEmpty()) { return null; } - return getDataTypeConverter().convertToDecimal(data); + return converter.convertToDecimal(data); default: - return getDataTypeConverter().convertFromStringToUTF8String(data); + return converter.convertFromStringToUTF8String(data); } } catch (NumberFormatException ex) { LOGGER.error("Problem while converting data type" + data); return null; } - } public static byte[] getBytesBasedOnDataTypeForNoDictionaryColumn(String dimensionValue, http://git-wip-us.apache.org/repos/asf/carbondata/blob/4e835095/core/src/main/java/org/apache/carbondata/core/util/comparator/Comparator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/comparator/Comparator.java b/core/src/main/java/org/apache/carbondata/core/util/comparator/Comparator.java index adce04f..1f43c7c 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/comparator/Comparator.java +++ b/core/src/main/java/org/apache/carbondata/core/util/comparator/Comparator.java @@ -42,6 +42,29 @@ public final class Comparator { return new ByteArraySerializableComparator(); } } + + /** + * create Comparator for Measure Datatype + * + * @param dataType + * @return + */ + public static SerializableComparator getComparatorByDataTypeForMeasure(DataType dataType) { + switch (dataType) { + case INT: + return new IntSerializableComparator(); + case SHORT: + return new ShortSerializableComparator(); + case LONG: + return new LongSerializableComparator(); + case DOUBLE: + return new DoubleSerializableComparator(); + case DECIMAL: + return new BigDecimalSerializableComparator(); + default: + throw new IllegalArgumentException("Unsupported data type"); + } + } } class ByteArraySerializableComparator implements SerializableComparator { @@ -52,6 +75,13 @@ class ByteArraySerializableComparator implements SerializableComparator { class IntSerializableComparator implements SerializableComparator { @Override public int compare(Object key1, Object key2) { + if (key1 == null && key2 == null) { + return 0; + } else if (key1 == null) { + return -1; + } else if (key2 == null) { + return 1; + } if ((int) key1 < (int) key2) { return -1; } else if ((int) key1 > (int) key2) { @@ -64,6 +94,13 @@ class IntSerializableComparator implements SerializableComparator { class ShortSerializableComparator implements SerializableComparator { @Override public int compare(Object key1, Object key2) { + if (key1 == null && key2 == null) { + return 0; + } else if (key1 == null) { + return -1; + } else if (key2 == null) { + return 1; + } if ((short) key1 < (short) key2) { return -1; } else if ((short) key1 > (short) key2) { @@ -76,6 +113,13 @@ class ShortSerializableComparator implements SerializableComparator { class DoubleSerializableComparator implements SerializableComparator { @Override public int compare(Object key1, Object key2) { + if (key1 == null && key2 == null) { + return 0; + } else if (key1 == null) { + return -1; + } else if (key2 == null) { + return 1; + } if ((double) key1 < (double) key2) { return -1; } else if ((double) key1 > (double) key2) { @@ -88,6 +132,13 @@ class DoubleSerializableComparator implements SerializableComparator { class LongSerializableComparator implements SerializableComparator { @Override public int compare(Object key1, Object key2) { + if (key1 == null && key2 == null) { + return 0; + } else if (key1 == null) { + return -1; + } else if (key2 == null) { + return 1; + } if ((long) key1 < (long) key2) { return -1; } else if ((long) key1 > (long) key2) { @@ -100,6 +151,13 @@ class LongSerializableComparator implements SerializableComparator { class BigDecimalSerializableComparator implements SerializableComparator { @Override public int compare(Object key1, Object key2) { + if (key1 == null && key2 == null) { + return 0; + } else if (key1 == null) { + return -1; + } else if (key2 == null) { + return 1; + } return ((BigDecimal) key1).compareTo((BigDecimal) key2); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/carbondata/blob/4e835095/core/src/main/java/org/apache/carbondata/core/util/comparator/SerializableComparator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/comparator/SerializableComparator.java b/core/src/main/java/org/apache/carbondata/core/util/comparator/SerializableComparator.java index df0d3e2..a8887eb 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/comparator/SerializableComparator.java +++ b/core/src/main/java/org/apache/carbondata/core/util/comparator/SerializableComparator.java @@ -19,6 +19,6 @@ package org.apache.carbondata.core.util.comparator; import java.io.Serializable; -public interface SerializableComparator extends Serializable { +public interface SerializableComparator extends java.util.Comparator, Serializable { int compare(Object key1, Object key2); } \ No newline at end of file
