subject:"\[GitHub\] carbondata pull request #1079\: \[WIP\]Measure Filter implementation"

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-07-02 Thread sounakr

Github user sounakr commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125199504
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
 ---
@@ -152,12 +261,31 @@ private BitSet 
setFilterdIndexToBitSet(DimensionColumnDataChunk dimensionColumnD
 
   public BitSet isScanRequired(byte[][] blkMaxVal, byte[][] blkMinVal) {
 BitSet bitSet = new BitSet(1);
-byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
-int columnIndex = dimColumnEvaluatorInfo.getColumnIndex();
-int blockIndex = 
segmentProperties.getDimensionOrdinalToBlockMapping().get(columnIndex);
+byte[][] filterValues = null;
+int columnIndex = 0;
+int blockIndex = 0;
+boolean isScanRequired = false;
+
+if (isDimensionPresentInCurrentBlock == true) {
+  filterValues = dimColumnExecuterInfo.getFilterKeys();
+  columnIndex = dimColumnEvaluatorInfo.getColumnIndex();
+  blockIndex = 
segmentProperties.getDimensionOrdinalToBlockMapping().get(columnIndex);
+  isScanRequired =
+  isScanRequired(blkMaxVal[blockIndex], blkMinVal[blockIndex], 
filterValues);
+
+} else if (isMeasurePresentInCurrentBlock) {
+  filterValues = msrColumnExecutorInfo.getFilterKeys();
+  columnIndex = msrColumnEvaluatorInfo.getColumnIndex();
+  // blockIndex =
+  // 
segmentProperties.getDimensionOrdinalToBlockMapping().get(columnIndex) + 
segmentProperties
+  // .getLastDimensionColOrdinal();
--- End diff --

Done


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-07-02 Thread sounakr

Github user sounakr commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125199254
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
 ---
@@ -18,56 +18,152 @@
 
 import java.io.IOException;
 import java.util.BitSet;
+import java.util.Comparator;
 
 import org.apache.carbondata.core.datastore.block.SegmentProperties;
 import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
 import 
org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
+import 
org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
+import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.scan.filter.FilterUtil;
+import 
org.apache.carbondata.core.scan.filter.partition.PartitionFilterUtil;
 import 
org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+import 
org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
 import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
 import org.apache.carbondata.core.util.BitSetGroup;
 import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.DataTypeUtil;
 
 public class ExcludeFilterExecuterImpl implements FilterExecuter {
 
   protected DimColumnResolvedFilterInfo dimColEvaluatorInfo;
   protected DimColumnExecuterFilterInfo dimColumnExecuterInfo;
+  protected MeasureColumnResolvedFilterInfo msrColumnEvaluatorInfo;
+  protected MeasureColumnExecuterFilterInfo msrColumnExecutorInfo;
   protected SegmentProperties segmentProperties;
+  protected boolean isDimensionPresentInCurrentBlock = false;
+  protected boolean isMeasurePresentInCurrentBlock = false;
   /**
* is dimension column data is natural sorted
*/
-  private boolean isNaturalSorted;
+  private boolean isNaturalSorted = false;
+
   public ExcludeFilterExecuterImpl(DimColumnResolvedFilterInfo 
dimColEvaluatorInfo,
-  SegmentProperties segmentProperties) {
-this.dimColEvaluatorInfo = dimColEvaluatorInfo;
-dimColumnExecuterInfo = new DimColumnExecuterFilterInfo();
+  MeasureColumnResolvedFilterInfo msrColumnEvaluatorInfo, 
SegmentProperties segmentProperties,
+  boolean isMeasure) {
 this.segmentProperties = segmentProperties;
-
FilterUtil.prepareKeysFromSurrogates(dimColEvaluatorInfo.getFilterValues(), 
segmentProperties,
-dimColEvaluatorInfo.getDimension(), dimColumnExecuterInfo);
-isNaturalSorted = 
dimColEvaluatorInfo.getDimension().isUseInvertedIndex() && dimColEvaluatorInfo
-.getDimension().isSortColumn();
+if (isMeasure == false) {
+  this.dimColEvaluatorInfo = dimColEvaluatorInfo;
+  dimColumnExecuterInfo = new DimColumnExecuterFilterInfo();
+
+  
FilterUtil.prepareKeysFromSurrogates(dimColEvaluatorInfo.getFilterValues(), 
segmentProperties,
+  dimColEvaluatorInfo.getDimension(), dimColumnExecuterInfo, null, 
null);
+  isDimensionPresentInCurrentBlock = true;
+  isNaturalSorted =
+  dimColEvaluatorInfo.getDimension().isUseInvertedIndex() && 
dimColEvaluatorInfo
+  .getDimension().isSortColumn();
+} else {
+  this.msrColumnEvaluatorInfo = msrColumnEvaluatorInfo;
+  msrColumnExecutorInfo = new MeasureColumnExecuterFilterInfo();
+  FilterUtil
+  
.prepareKeysFromSurrogates(msrColumnEvaluatorInfo.getFilterValues(), 
segmentProperties,
+  null, null, msrColumnEvaluatorInfo.getMeasure(), 
msrColumnExecutorInfo);
+  isMeasurePresentInCurrentBlock = true;
+}
+
   }
 
   @Override public BitSetGroup applyFilter(BlocksChunkHolder 
blockChunkHolder) throws IOException {
-int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
-.get(dimColEvaluatorInfo.getColumnIndex());
-if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
-  blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = 
blockChunkHolder.getDataBlock()
-  .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+if (isDimensionPresentInCurrentBlock == true) {
+  int blockIndex = 
segmentProperties.getDimensionOrdinalToBlockMapping()
+  .get(dimColEvaluatorInfo.getColumnIndex());
+  if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) 
{
+blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = 
blockChunkHolder.getDataBlock()
+.getDimensionChunk(blockChunkHolder.getFileReader(), 
blockIndex);
+

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-07-02 Thread sounakr

Github user sounakr commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125199288
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
 ---
@@ -17,65 +17,174 @@
 package org.apache.carbondata.core.scan.filter.executer;
 
 import java.io.IOException;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
 import java.util.BitSet;
+import java.util.Comparator;
 
 import org.apache.carbondata.core.datastore.block.SegmentProperties;
 import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
 import 
org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
+import 
org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
+import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.scan.filter.FilterUtil;
+import 
org.apache.carbondata.core.scan.filter.partition.PartitionFilterUtil;
 import 
org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+import 
org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
 import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
 import org.apache.carbondata.core.util.BitSetGroup;
 import org.apache.carbondata.core.util.ByteUtil;
 import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.DataTypeUtil;
 
 public class IncludeFilterExecuterImpl implements FilterExecuter {
 
   protected DimColumnResolvedFilterInfo dimColumnEvaluatorInfo;
   protected DimColumnExecuterFilterInfo dimColumnExecuterInfo;
+  protected MeasureColumnResolvedFilterInfo msrColumnEvaluatorInfo;
+  protected MeasureColumnExecuterFilterInfo msrColumnExecutorInfo;
   protected SegmentProperties segmentProperties;
+  protected boolean isDimensionPresentInCurrentBlock = false;
+  protected boolean isMeasurePresentInCurrentBlock = false;
--- End diff --

Done.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-07-02 Thread sounakr

Github user sounakr commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125199276
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
 ---
@@ -17,65 +17,174 @@
 package org.apache.carbondata.core.scan.filter.executer;
 
 import java.io.IOException;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
 import java.util.BitSet;
+import java.util.Comparator;
 
 import org.apache.carbondata.core.datastore.block.SegmentProperties;
 import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
 import 
org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
+import 
org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
+import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.scan.filter.FilterUtil;
+import 
org.apache.carbondata.core.scan.filter.partition.PartitionFilterUtil;
 import 
org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+import 
org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
 import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
 import org.apache.carbondata.core.util.BitSetGroup;
 import org.apache.carbondata.core.util.ByteUtil;
 import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.DataTypeUtil;
 
 public class IncludeFilterExecuterImpl implements FilterExecuter {
 
   protected DimColumnResolvedFilterInfo dimColumnEvaluatorInfo;
   protected DimColumnExecuterFilterInfo dimColumnExecuterInfo;
+  protected MeasureColumnResolvedFilterInfo msrColumnEvaluatorInfo;
+  protected MeasureColumnExecuterFilterInfo msrColumnExecutorInfo;
   protected SegmentProperties segmentProperties;
+  protected boolean isDimensionPresentInCurrentBlock = false;
+  protected boolean isMeasurePresentInCurrentBlock = false;
   /**
* is dimension column data is natural sorted
*/
-  private boolean isNaturalSorted;
+  private boolean isNaturalSorted = false;
--- End diff --

Done.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-07-02 Thread sounakr

Github user sounakr commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125199168
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
 ---
@@ -18,56 +18,152 @@
 
 import java.io.IOException;
 import java.util.BitSet;
+import java.util.Comparator;
 
 import org.apache.carbondata.core.datastore.block.SegmentProperties;
 import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
 import 
org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
+import 
org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
+import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.scan.filter.FilterUtil;
+import 
org.apache.carbondata.core.scan.filter.partition.PartitionFilterUtil;
 import 
org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+import 
org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
 import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
 import org.apache.carbondata.core.util.BitSetGroup;
 import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.DataTypeUtil;
 
 public class ExcludeFilterExecuterImpl implements FilterExecuter {
 
   protected DimColumnResolvedFilterInfo dimColEvaluatorInfo;
   protected DimColumnExecuterFilterInfo dimColumnExecuterInfo;
+  protected MeasureColumnResolvedFilterInfo msrColumnEvaluatorInfo;
+  protected MeasureColumnExecuterFilterInfo msrColumnExecutorInfo;
   protected SegmentProperties segmentProperties;
+  protected boolean isDimensionPresentInCurrentBlock = false;
+  protected boolean isMeasurePresentInCurrentBlock = false;
--- End diff --

Done, removed.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-07-02 Thread sounakr

Github user sounakr commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125198925
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
 ---
@@ -18,56 +18,152 @@
 
 import java.io.IOException;
 import java.util.BitSet;
+import java.util.Comparator;
 
 import org.apache.carbondata.core.datastore.block.SegmentProperties;
 import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
 import 
org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
+import 
org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
+import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.scan.filter.FilterUtil;
+import 
org.apache.carbondata.core.scan.filter.partition.PartitionFilterUtil;
 import 
org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+import 
org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
 import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
 import org.apache.carbondata.core.util.BitSetGroup;
 import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.DataTypeUtil;
 
 public class ExcludeFilterExecuterImpl implements FilterExecuter {
 
   protected DimColumnResolvedFilterInfo dimColEvaluatorInfo;
   protected DimColumnExecuterFilterInfo dimColumnExecuterInfo;
+  protected MeasureColumnResolvedFilterInfo msrColumnEvaluatorInfo;
+  protected MeasureColumnExecuterFilterInfo msrColumnExecutorInfo;
   protected SegmentProperties segmentProperties;
+  protected boolean isDimensionPresentInCurrentBlock = false;
+  protected boolean isMeasurePresentInCurrentBlock = false;
   /**
* is dimension column data is natural sorted
*/
-  private boolean isNaturalSorted;
+  private boolean isNaturalSorted = false;
+
   public ExcludeFilterExecuterImpl(DimColumnResolvedFilterInfo 
dimColEvaluatorInfo,
-  SegmentProperties segmentProperties) {
-this.dimColEvaluatorInfo = dimColEvaluatorInfo;
-dimColumnExecuterInfo = new DimColumnExecuterFilterInfo();
+  MeasureColumnResolvedFilterInfo msrColumnEvaluatorInfo, 
SegmentProperties segmentProperties,
+  boolean isMeasure) {
 this.segmentProperties = segmentProperties;
-
FilterUtil.prepareKeysFromSurrogates(dimColEvaluatorInfo.getFilterValues(), 
segmentProperties,
-dimColEvaluatorInfo.getDimension(), dimColumnExecuterInfo);
-isNaturalSorted = 
dimColEvaluatorInfo.getDimension().isUseInvertedIndex() && dimColEvaluatorInfo
-.getDimension().isSortColumn();
+if (isMeasure == false) {
--- End diff --

Done


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-07-02 Thread sounakr

Github user sounakr commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125198825
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java ---
@@ -1042,12 +1144,17 @@ public static FilterExecuter getFilterExecuterTree(
* @param dimension
* @param dimColumnExecuterInfo
*/
-  public static void prepareKeysFromSurrogates(DimColumnFilterInfo 
filterValues,
+  public static void prepareKeysFromSurrogates(ColumnFilterInfo 
filterValues,
   SegmentProperties segmentProperties, CarbonDimension dimension,
-  DimColumnExecuterFilterInfo dimColumnExecuterInfo) {
-byte[][] keysBasedOnFilter = getKeyArray(filterValues, dimension, 
segmentProperties);
-dimColumnExecuterInfo.setFilterKeys(keysBasedOnFilter);
-
+  DimColumnExecuterFilterInfo dimColumnExecuterInfo, CarbonMeasure 
measures,
+  MeasureColumnExecuterFilterInfo msrColumnExecuterInfo) {
+if (null != measures) {
--- End diff --

This if check is required in order to setFilterKeys in respective measures 
or dimensions. 


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-07-02 Thread sounakr

Github user sounakr commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125191929
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java ---
@@ -113,6 +115,143 @@ public static Object 
getMeasureValueBasedOnDataType(String msrValue, DataType da
 }
   }
 
+  public static Object getMeasureObjectFromDataType(byte[] data, DataType 
dataType) {
+ByteBuffer bb = ByteBuffer.wrap(data);
--- End diff --

Done


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-07-02 Thread sounakr

Github user sounakr commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125191810
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java ---
@@ -395,6 +440,58 @@ public static DimColumnFilterInfo 
getNoDictionaryValKeyMemberForFilter(
   }
 
   /**
+   * This method will get the no dictionary data based on filters and same
+   * will be in ColumnFilterInfo
+   *
+   * @param evaluateResultListFinal
+   * @param isIncludeFilter
+   * @return ColumnFilterInfo
+   */
+  public static ColumnFilterInfo getMeasureValKeyMemberForFilter(
+  List evaluateResultListFinal, boolean isIncludeFilter, 
DataType dataType,
+  CarbonMeasure carbonMeasure) throws FilterUnsupportedException {
+List filterValuesList = new ArrayList(20);
+String result = null;
+try {
+  int length = evaluateResultListFinal.size();
+  for (int i = 0; i < length; i++) {
+result = evaluateResultListFinal.get(i);
+if (CarbonCommonConstants.MEMBER_DEFAULT_VAL.equals(result)) {
+  filterValuesList.add(new byte[0]);
+  continue;
+}
+// TODO have to understand what method to be used for measures.
+// filterValuesList
+//  
.add(DataTypeUtil.getBytesBasedOnDataTypeForNoDictionaryColumn(result, 
dataType));
+
+filterValuesList
+.add(DataTypeUtil.getMeasureByteArrayBasedOnDataTypes(result, 
dataType, carbonMeasure));
--- End diff --

Currently we are storing filter keys of measures in byte array format. In 
next optimization phase will change to Object array of respective type to avoid 
conversion. 


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-07-02 Thread sounakr

Github user sounakr commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125191770
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java ---
@@ -113,6 +115,143 @@ public static Object 
getMeasureValueBasedOnDataType(String msrValue, DataType da
 }
   }
 
+  public static Object getMeasureObjectFromDataType(byte[] data, DataType 
dataType) {
+ByteBuffer bb = ByteBuffer.wrap(data);
+switch (dataType) {
+  case SHORT:
+  case INT:
+  case LONG:
+return bb.getLong();
+  case DECIMAL:
+return byteToBigDecimal(data);
+  default:
+return bb.getDouble();
+}
+  }
+
+  /**
+   * This method will convert a given ByteArray to its specific type
+   *
+   * @param msrValue
+   * @param dataType
+   * @param carbonMeasure
+   * @return
+   */
+  //  public static byte[] getMeasureByteArrayBasedOnDataType(String 
msrValue, DataType dataType,
+  //  CarbonMeasure carbonMeasure) {
+  //switch (dataType) {
+  //  case DECIMAL:
+  //BigDecimal bigDecimal =
+  //new 
BigDecimal(msrValue).setScale(carbonMeasure.getScale(), RoundingMode.HALF_UP);
+  //   return ByteUtil.toBytes(normalizeDecimalValue(bigDecimal, 
carbonMeasure.getPrecision()));
+  //  case SHORT:
+  //return ByteUtil.toBytes((Short.parseShort(msrValue)));
+  //  case INT:
+  //return ByteUtil.toBytes(Integer.parseInt(msrValue));
+  //  case LONG:
+  //return ByteUtil.toBytes(Long.valueOf(msrValue));
+  //  default:
+  //Double parsedValue = Double.valueOf(msrValue);
+  //if (Double.isInfinite(parsedValue) || 
Double.isNaN(parsedValue)) {
+  //  return null;
+  //}
+  //return ByteUtil.toBytes(parsedValue);
+  //}
+  //  }
--- End diff --

Done


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-07-02 Thread sounakr

Github user sounakr commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125191772
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java ---
@@ -113,6 +115,143 @@ public static Object 
getMeasureValueBasedOnDataType(String msrValue, DataType da
 }
   }
 
+  public static Object getMeasureObjectFromDataType(byte[] data, DataType 
dataType) {
+ByteBuffer bb = ByteBuffer.wrap(data);
+switch (dataType) {
+  case SHORT:
+  case INT:
+  case LONG:
+return bb.getLong();
+  case DECIMAL:
+return byteToBigDecimal(data);
+  default:
+return bb.getDouble();
+}
+  }
+
+  /**
+   * This method will convert a given ByteArray to its specific type
+   *
+   * @param msrValue
+   * @param dataType
+   * @param carbonMeasure
+   * @return
+   */
+  //  public static byte[] getMeasureByteArrayBasedOnDataType(String 
msrValue, DataType dataType,
+  //  CarbonMeasure carbonMeasure) {
+  //switch (dataType) {
+  //  case DECIMAL:
+  //BigDecimal bigDecimal =
+  //new 
BigDecimal(msrValue).setScale(carbonMeasure.getScale(), RoundingMode.HALF_UP);
+  //   return ByteUtil.toBytes(normalizeDecimalValue(bigDecimal, 
carbonMeasure.getPrecision()));
+  //  case SHORT:
+  //return ByteUtil.toBytes((Short.parseShort(msrValue)));
+  //  case INT:
+  //return ByteUtil.toBytes(Integer.parseInt(msrValue));
+  //  case LONG:
+  //return ByteUtil.toBytes(Long.valueOf(msrValue));
+  //  default:
+  //Double parsedValue = Double.valueOf(msrValue);
+  //if (Double.isInfinite(parsedValue) || 
Double.isNaN(parsedValue)) {
+  //  return null;
+  //}
+  //return ByteUtil.toBytes(parsedValue);
+  //}
+  //  }
+  public static byte[] getMeasureByteArrayBasedOnDataTypes(String 
msrValue, DataType dataType,
+  CarbonMeasure carbonMeasure) {
+ByteBuffer b;
+switch (dataType) {
+  case BYTE:
+  case SHORT:
+  case INT:
+  case LONG:
+b = ByteBuffer.allocate(8);
+b.putLong(Long.valueOf(msrValue));
+b.flip();
+return b.array();
+  case DOUBLE:
+b = ByteBuffer.allocate(8);
+b.putDouble(Double.valueOf(msrValue));
+b.flip();
+return b.array();
+  case DECIMAL:
+BigDecimal bigDecimal =
+new BigDecimal(msrValue).setScale(carbonMeasure.getScale(), 
RoundingMode.HALF_UP);
+return DataTypeUtil
+.bigDecimalToByte(normalizeDecimalValue(bigDecimal, 
carbonMeasure.getPrecision()));
+  default:
+throw new IllegalArgumentException("Invalid data type: " + 
dataType);
+}
+  }
+
+  /**
+   * This method will convert a given ByteArray to its specific type
+   *
+   * @param msrValue
+   * @param dataType
+   * @param carbonMeasure
+   * @return
+   */
+  public static byte[] getMeasureByteArrayBasedOnDataType(ColumnPage 
measurePage, int index,
--- End diff --

Done


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-07-02 Thread sounakr

Github user sounakr commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125191734
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
 ---
@@ -74,80 +87,205 @@ private void ifDefaultValueMatchesFilter() {
   }
 }
   }
+} else if (!msrColEvalutorInfoList.isEmpty() && 
!isMeasurePresentInCurrentBlock[0]) {
+  CarbonMeasure measure = 
this.msrColEvalutorInfoList.get(0).getMeasure();
+  byte[] defaultValue = measure.getDefaultValue();
+  if (null != defaultValue) {
+for (int k = 0; k < filterRangeValues.length; k++) {
+  int maxCompare =
+  
ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterRangeValues[k], defaultValue);
+  if (maxCompare < 0) {
+isDefaultValuePresentInFilter = true;
+break;
+  }
+}
+  }
 }
   }
 
   @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] 
blockMinValue) {
 BitSet bitSet = new BitSet(1);
-boolean isScanRequired =
-isScanRequired(blockMaxValue[dimensionBlocksIndex[0]], 
filterRangeValues);
+boolean isScanRequired = false;
+byte[] maxValue = null;
+if (isMeasurePresentInCurrentBlock[0] || 
isDimensionPresentInCurrentBlock[0]) {
+  if (isMeasurePresentInCurrentBlock[0]) {
+maxValue = blockMaxValue[measureBlocksIndex[0] + 
lastDimensionColOrdinal];
+isScanRequired =
+isScanRequired(maxValue, filterRangeValues, 
msrColEvalutorInfoList.get(0).getType());
+  } else {
+maxValue = blockMaxValue[dimensionBlocksIndex[0]];
+isScanRequired = isScanRequired(maxValue, filterRangeValues);
+  }
+} else {
+  isScanRequired = isDefaultValuePresentInFilter;
+}
+
 if (isScanRequired) {
   bitSet.set(0);
 }
 return bitSet;
   }
 
+
   private boolean isScanRequired(byte[] blockMaxValue, byte[][] 
filterValues) {
 boolean isScanRequired = false;
-if (isDimensionPresentInCurrentBlock[0]) {
-  for (int k = 0; k < filterValues.length; k++) {
-// filter value should be in range of max and min value i.e
-// max>filtervalue>min
-// so filter-max should be negative
-int maxCompare = 
ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMaxValue);
-// if any filter value is in range than this block needs to be
-// scanned means always less than block max range.
-if (maxCompare < 0) {
-  isScanRequired = true;
-  break;
-}
+for (int k = 0; k < filterValues.length; k++) {
+  // filter value should be in range of max and min value i.e
+  // max>filtervalue>min
+  // so filter-max should be negative
+  int maxCompare = 
ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMaxValue);
+  // if any filter value is in range than this block needs to be
+  // scanned less than equal to max range.
+  if (maxCompare < 0) {
+isScanRequired = true;
+break;
   }
-} else {
-  isScanRequired = isDefaultValuePresentInFilter;
 }
 return isScanRequired;
   }
 
+  private boolean isScanRequired(byte[] maxValue, byte[][] filterValue,
+  DataType dataType) {
+for (int i = 0; i < filterValue.length; i++) {
+  if (filterValue[i].length == 0 || maxValue.length == 0) {
+return isScanRequired(maxValue, filterValue);
+  }
+  switch (dataType) {
--- End diff --

In Line 150 is a special Null Value case, rest of the cases comparator is 
being used. 


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-07-02 Thread sounakr

Github user sounakr commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125191629
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
 ---
@@ -74,80 +87,205 @@ private void ifDefaultValueMatchesFilter() {
   }
 }
   }
+} else if (!msrColEvalutorInfoList.isEmpty() && 
!isMeasurePresentInCurrentBlock[0]) {
+  CarbonMeasure measure = 
this.msrColEvalutorInfoList.get(0).getMeasure();
+  byte[] defaultValue = measure.getDefaultValue();
+  if (null != defaultValue) {
+for (int k = 0; k < filterRangeValues.length; k++) {
+  int maxCompare =
+  
ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterRangeValues[k], defaultValue);
--- End diff --

Currently Filter Keys are in ByteArray and values for restructuring is also 
same. Is this still required.  


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-07-02 Thread sounakr

Github user sounakr commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125191588
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/scan/filter/executer/MeasureColumnExecuterFilterInfo.java
 ---
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.scan.filter.executer;
+
+public class MeasureColumnExecuterFilterInfo {
+
+  byte[][] filterKeys;
--- End diff --

In Current Implementation kept filterkeys as Byte Array to keep it simple 
and in sync with dimention array. During actual comparision the filter keys are 
converted back to Objects and compared. In next phase optimization will change 
the Filter Keys to hold objects. 


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-07-02 Thread sounakr

Github user sounakr commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125191432
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/scan/filter/partition/PartitionFilterUtil.java
 ---
@@ -76,24 +99,25 @@ public static Comparator 
getComparatorByDataType(DataType dataType) {
 
   static class DoubleComparator implements Comparator {
 @Override public int compare(Object key1, Object key2) {
-  double result = (double) key1 - (double) key2;
-  if (result < 0) {
+  double key1Double1 = (double)key1;
--- End diff --

There is a scenario is a variable is a negative one the Key1 - Key2 wont 
give proper output. Better to check greater or less than operator. 


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-07-02 Thread sounakr

Github user sounakr commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125191402
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/scan/filter/partition/PartitionFilterUtil.java
 ---
@@ -107,6 +131,12 @@ public static Comparator 
getComparatorByDataType(DataType dataType) {
 }
   }
 
+  static class DecimalComparator implements Comparator {
--- End diff --

Done. Removed


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-07-02 Thread sounakr

Github user sounakr commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125191383
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
 ---
@@ -44,16 +44,22 @@
   protected boolean isExpressionResolve;
   protected boolean isIncludeFilter;
   private DimColumnResolvedFilterInfo dimColResolvedFilterInfo;
+  private MeasureColumnResolvedFilterInfo msrColResolvedFilterInfo;
   private AbsoluteTableIdentifier tableIdentifier;
+  private boolean isMeasure;
--- End diff --

Done. Removed


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-07-02 Thread sounakr

Github user sounakr commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125191362
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
 ---
@@ -198,21 +237,31 @@ public AbsoluteTableIdentifier getTableIdentifier() {
*/
   public void getStartKey(SegmentProperties segmentProperties, long[] 
startKey,
   SortedMap setOfStartKeyByteArray, List 
startKeyList) {
-
FilterUtil.getStartKey(dimColResolvedFilterInfo.getDimensionResolvedFilterInstance(),
-segmentProperties, startKey, startKeyList);
-
FilterUtil.getStartKeyForNoDictionaryDimension(dimColResolvedFilterInfo,
-segmentProperties, setOfStartKeyByteArray);
+if (null != dimColResolvedFilterInfo) {
+  
FilterUtil.getStartKey(dimColResolvedFilterInfo.getDimensionResolvedFilterInstance(),
+  segmentProperties, startKey, startKeyList);
+  
FilterUtil.getStartKeyForNoDictionaryDimension(dimColResolvedFilterInfo, 
segmentProperties,
+  setOfStartKeyByteArray);
+}
+// else {
+//  
FilterUtil.getStartKey(dimColResolvedFilterInfo.getDimensionResolvedFilterInstance(),
+//  segmentProperties, startKey, startKeyList);
+//  
FilterUtil.getStartKeyForNoDictionaryDimension(dimColResolvedFilterInfo, 
segmentProperties,
+//  setOfStartKeyByteArray);
+//}
--- End diff --

Done.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-07-02 Thread sounakr

Github user sounakr commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125191345
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java ---
@@ -395,6 +440,58 @@ public static DimColumnFilterInfo 
getNoDictionaryValKeyMemberForFilter(
   }
 
   /**
+   * This method will get the no dictionary data based on filters and same
+   * will be in ColumnFilterInfo
+   *
+   * @param evaluateResultListFinal
+   * @param isIncludeFilter
+   * @return ColumnFilterInfo
+   */
+  public static ColumnFilterInfo getMeasureValKeyMemberForFilter(
+  List evaluateResultListFinal, boolean isIncludeFilter, 
DataType dataType,
+  CarbonMeasure carbonMeasure) throws FilterUnsupportedException {
+List filterValuesList = new ArrayList(20);
+String result = null;
+try {
+  int length = evaluateResultListFinal.size();
+  for (int i = 0; i < length; i++) {
+result = evaluateResultListFinal.get(i);
+if (CarbonCommonConstants.MEMBER_DEFAULT_VAL.equals(result)) {
+  filterValuesList.add(new byte[0]);
+  continue;
+}
+// TODO have to understand what method to be used for measures.
+// filterValuesList
+//  
.add(DataTypeUtil.getBytesBasedOnDataTypeForNoDictionaryColumn(result, 
dataType));
+
+filterValuesList
+.add(DataTypeUtil.getMeasureByteArrayBasedOnDataTypes(result, 
dataType, carbonMeasure));
+
+  }
+} catch (Throwable ex) {
+  throw new FilterUnsupportedException("Unsupported Filter condition: 
" + result, ex);
+}
+
+Comparator filterMeasureComaparator = new Comparator() 
{
+
+  @Override public int compare(byte[] filterMember1, byte[] 
filterMember2) {
+// TODO Auto-generated method stub
+return ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterMember1, 
filterMember2);
--- End diff --

We are converting String dataTypes into Bytes array and then saving into 
filterValueList. From populateFilterResolvedInfo itself we convert all dataType 
to strings and then pass it along. We may have to rectify and pass on actual 
datatype from populateFilterResolvedInfo. 

But all filterValue Comparision in measures are currently sequential, so 
there no chance of getting a wrong result. i.e. in Include, RowLevelLessThan, 
RowLevelLessThanEqual, RowLevelGrtThanEqual, RowLevelGrtThan. Also Range is not 
implemented yet for measures where 2 filter values should be in ascending 
order.  In the next stage optimization we can hold the filter values in actual 
datatype and have comparator for each. This was comparision will be proper and 
we dont have to convert to datatype to byte and again back to object while 
doing actual comparision, we can carry object all along.  


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-07-02 Thread sounakr

Github user sounakr commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125190940
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java ---
@@ -395,6 +440,58 @@ public static DimColumnFilterInfo 
getNoDictionaryValKeyMemberForFilter(
   }
 
   /**
+   * This method will get the no dictionary data based on filters and same
+   * will be in ColumnFilterInfo
+   *
+   * @param evaluateResultListFinal
+   * @param isIncludeFilter
+   * @return ColumnFilterInfo
+   */
+  public static ColumnFilterInfo getMeasureValKeyMemberForFilter(
+  List evaluateResultListFinal, boolean isIncludeFilter, 
DataType dataType,
+  CarbonMeasure carbonMeasure) throws FilterUnsupportedException {
+List filterValuesList = new ArrayList(20);
+String result = null;
+try {
+  int length = evaluateResultListFinal.size();
+  for (int i = 0; i < length; i++) {
+result = evaluateResultListFinal.get(i);
+if (CarbonCommonConstants.MEMBER_DEFAULT_VAL.equals(result)) {
+  filterValuesList.add(new byte[0]);
+  continue;
+}
+// TODO have to understand what method to be used for measures.
+// filterValuesList
+//  
.add(DataTypeUtil.getBytesBasedOnDataTypeForNoDictionaryColumn(result, 
dataType));
--- End diff --

Done. Removed.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-07-02 Thread sounakr

Github user sounakr commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125190884
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java ---
@@ -180,9 +185,27 @@ private static FilterExecuter createFilterExecuterTree(
* @return
*/
   private static FilterExecuter getIncludeFilterExecuter(
-  DimColumnResolvedFilterInfo dimColResolvedFilterInfo, 
SegmentProperties segmentProperties) {
-
-if (dimColResolvedFilterInfo.getDimension().isColumnar()) {
+  DimColumnResolvedFilterInfo dimColResolvedFilterInfo,
+  MeasureColumnResolvedFilterInfo msrColResolvedFilterInfo,
+  SegmentProperties segmentProperties) {
+if (null != msrColResolvedFilterInfo && 
msrColResolvedFilterInfo.getMeasure().isColumnar()) {
+  CarbonMeasure measuresFromCurrentBlock = segmentProperties
+  
.getMeasureFromCurrentBlock(msrColResolvedFilterInfo.getMeasure().getColumnId());
+  if (null != measuresFromCurrentBlock) {
+// update dimension and column index according to the dimension 
position in current block
+MeasureColumnResolvedFilterInfo msrColResolvedFilterInfoCopyObject 
=
+msrColResolvedFilterInfo.getCopyObject();
+
msrColResolvedFilterInfoCopyObject.setMeasure(measuresFromCurrentBlock);
+
msrColResolvedFilterInfoCopyObject.setColumnIndex(measuresFromCurrentBlock.getOrdinal());
+
msrColResolvedFilterInfoCopyObject.setType(measuresFromCurrentBlock.getDataType());
+return new IncludeFilterExecuterImpl(null, 
msrColResolvedFilterInfoCopyObject,
+segmentProperties, true);
+  } else {
+return new 
RestructureIncludeFilterExecutorImpl(dimColResolvedFilterInfo,
+msrColResolvedFilterInfo, segmentProperties, true);
+  }
+}
--- End diff --

No, This whole block belongs to Measure. Line 220 points to Restructure in 
case of dimentions.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-07-02 Thread sounakr

Github user sounakr commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125190853
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java ---
@@ -180,9 +185,27 @@ private static FilterExecuter createFilterExecuterTree(
* @return
*/
   private static FilterExecuter getIncludeFilterExecuter(
-  DimColumnResolvedFilterInfo dimColResolvedFilterInfo, 
SegmentProperties segmentProperties) {
-
-if (dimColResolvedFilterInfo.getDimension().isColumnar()) {
+  DimColumnResolvedFilterInfo dimColResolvedFilterInfo,
+  MeasureColumnResolvedFilterInfo msrColResolvedFilterInfo,
+  SegmentProperties segmentProperties) {
+if (null != msrColResolvedFilterInfo && 
msrColResolvedFilterInfo.getMeasure().isColumnar()) {
--- End diff --

Done. Removed.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-07-02 Thread sounakr

Github user sounakr commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125190787
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
 ---
@@ -137,6 +137,8 @@
*/
   private int numberOfNoDictSortColumns;
 
+  private int lastDimensionColumnOrdinal;
--- End diff --

Removed. 


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-07-02 Thread sounakr

Github user sounakr commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125190753
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/ColumnPageStatsVO.java
 ---
@@ -56,9 +65,7 @@ public ColumnPageStatsVO(DataType dataType) {
 nonExistValue = Double.MIN_VALUE;
 break;
   case DECIMAL:
-max = new BigDecimal(Double.MIN_VALUE);
-min = new BigDecimal(Double.MAX_VALUE);
-nonExistValue = new BigDecimal(Double.MIN_VALUE);
+this.zeroDecimal = new BigDecimal(0);
--- End diff --

Done


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-06-30 Thread ravipesala

Github user ravipesala commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125154345
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
 ---
@@ -198,21 +237,31 @@ public AbsoluteTableIdentifier getTableIdentifier() {
*/
   public void getStartKey(SegmentProperties segmentProperties, long[] 
startKey,
   SortedMap setOfStartKeyByteArray, List 
startKeyList) {
-
FilterUtil.getStartKey(dimColResolvedFilterInfo.getDimensionResolvedFilterInstance(),
-segmentProperties, startKey, startKeyList);
-
FilterUtil.getStartKeyForNoDictionaryDimension(dimColResolvedFilterInfo,
-segmentProperties, setOfStartKeyByteArray);
+if (null != dimColResolvedFilterInfo) {
+  
FilterUtil.getStartKey(dimColResolvedFilterInfo.getDimensionResolvedFilterInstance(),
+  segmentProperties, startKey, startKeyList);
+  
FilterUtil.getStartKeyForNoDictionaryDimension(dimColResolvedFilterInfo, 
segmentProperties,
+  setOfStartKeyByteArray);
+}
+// else {
+//  
FilterUtil.getStartKey(dimColResolvedFilterInfo.getDimensionResolvedFilterInstance(),
+//  segmentProperties, startKey, startKeyList);
+//  
FilterUtil.getStartKeyForNoDictionaryDimension(dimColResolvedFilterInfo, 
segmentProperties,
+//  setOfStartKeyByteArray);
+//}
--- End diff --

remove commented code


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-06-30 Thread ravipesala

Github user ravipesala commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125154321
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/scan/filter/partition/PartitionFilterUtil.java
 ---
@@ -107,6 +131,12 @@ public static Comparator 
getComparatorByDataType(DataType dataType) {
 }
   }
 
+  static class DecimalComparator implements Comparator {
--- End diff --

what is the use of this comparator? Please remove if not used


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-06-30 Thread ravipesala

Github user ravipesala commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125154292
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/scan/filter/partition/PartitionFilterUtil.java
 ---
@@ -76,24 +99,25 @@ public static Comparator 
getComparatorByDataType(DataType dataType) {
 
   static class DoubleComparator implements Comparator {
 @Override public int compare(Object key1, Object key2) {
-  double result = (double) key1 - (double) key2;
-  if (result < 0) {
+  double key1Double1 = (double)key1;
--- End diff --

Why need to change this logic? old logic seems fine right


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-06-30 Thread ravipesala

Github user ravipesala commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125154262
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
 ---
@@ -77,72 +89,188 @@ private void ifDefaultValueMatchesFilter() {
   }
 }
   }
+} else if (!msrColEvalutorInfoList.isEmpty() && 
!isMeasurePresentInCurrentBlock[0]) {
+  CarbonMeasure measure = 
this.msrColEvalutorInfoList.get(0).getMeasure();
+  byte[] defaultValue = measure.getDefaultValue();
+  if (null != defaultValue) {
+for (int k = 0; k < filterRangeValues.length; k++) {
+  int maxCompare =
+  
ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterRangeValues[k], defaultValue);
+  if (maxCompare > 0) {
+isDefaultValuePresentInFilter = true;
+break;
+  }
+}
+  }
 }
   }
 
   @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] 
blockMinValue) {
 BitSet bitSet = new BitSet(1);
-boolean isScanRequired =
-isScanRequired(blockMinValue[dimensionBlocksIndex[0]], 
filterRangeValues);
+byte[] minValue = null;
+boolean isScanRequired = false;
+if (isMeasurePresentInCurrentBlock[0] || 
isDimensionPresentInCurrentBlock[0]) {
+  if (isMeasurePresentInCurrentBlock[0]) {
+minValue = blockMinValue[measureBlocksIndex[0] + 
lastDimensionColOrdinal];
+isScanRequired =
+isScanRequired(minValue, filterRangeValues, 
msrColEvalutorInfoList.get(0).getType());
+  } else {
+minValue = blockMinValue[dimensionBlocksIndex[0]];
+isScanRequired = isScanRequired(minValue, filterRangeValues);
+  }
+} else {
+  isScanRequired = isDefaultValuePresentInFilter;
+}
 if (isScanRequired) {
   bitSet.set(0);
 }
 return bitSet;
   }
 
+
   private boolean isScanRequired(byte[] blockMinValue, byte[][] 
filterValues) {
 boolean isScanRequired = false;
-if (isDimensionPresentInCurrentBlock[0]) {
-  for (int k = 0; k < filterValues.length; k++) {
-// and filter-min should be positive
-int minCompare = 
ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMinValue);
+for (int k = 0; k < filterValues.length; k++) {
+  // and filter-min should be positive
+  int minCompare = 
ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMinValue);
 
-// if any filter applied is not in range of min and max of block
-// then since its a less than fiter validate whether the block
-// min range is less  than applied filter member
-if (minCompare > 0) {
-  isScanRequired = true;
-  break;
-}
+  // if any filter applied is not in range of min and max of block
+  // then since its a less than equal to fiter validate whether the 
block
+  // min range is less than equal to applied filter member
+  if (minCompare > 0) {
+isScanRequired = true;
+break;
   }
-} else {
-  isScanRequired = isDefaultValuePresentInFilter;
 }
 return isScanRequired;
   }
 
+  private boolean isScanRequired(byte[] minValue, byte[][] filterValue,
+  DataType dataType) {
+for (int i = 0; i < filterValue.length; i++) {
+  if (filterValue[i].length == 0 || minValue.length == 0) {
+return isScanRequired(minValue, filterValue);
+  }
+  switch (dataType) {
--- End diff --

Use existing methods of `DataTypeUtil` and comparator here


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-06-30 Thread ravipesala

Github user ravipesala commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125154233
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
 ---
@@ -91,57 +129,147 @@ private void ifDefaultValueMatchesFilter() {
 
   private boolean isScanRequired(byte[] blockMinValue, byte[][] 
filterValues) {
 boolean isScanRequired = false;
-if (isDimensionPresentInCurrentBlock[0]) {
-  for (int k = 0; k < filterValues.length; k++) {
-// and filter-min should be positive
-int minCompare = 
ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMinValue);
+for (int k = 0; k < filterValues.length; k++) {
+  // and filter-min should be positive
+  int minCompare = 
ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMinValue);
 
-// if any filter applied is not in range of min and max of block
-// then since its a less than equal to fiter validate whether the 
block
-// min range is less than equal to applied filter member
-if (minCompare >= 0) {
-  isScanRequired = true;
-  break;
-}
+  // if any filter applied is not in range of min and max of block
+  // then since its a less than equal to fiter validate whether the 
block
+  // min range is less than equal to applied filter member
+  if (minCompare >= 0) {
+isScanRequired = true;
+break;
   }
-} else {
-  isScanRequired = isDefaultValuePresentInFilter;
 }
 return isScanRequired;
   }
 
+  private boolean isScanRequired(byte[] minValue, byte[][] filterValue,
+  DataType dataType) {
+for (int i = 0; i < filterValue.length; i++) {
+  if (filterValue[i].length == 0 || minValue.length == 0) {
+return isScanRequired(minValue, filterValue);
+  }
+  switch (dataType) {
--- End diff --

Use existing methods of `DataTypeUtil` and comparator here


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-06-30 Thread ravipesala

Github user ravipesala commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125154226
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
 ---
@@ -91,67 +131,167 @@ private void ifDefaultValueMatchesFilter() {
 
   private boolean isScanRequired(byte[] blockMaxValue, byte[][] 
filterValues) {
 boolean isScanRequired = false;
-if (isDimensionPresentInCurrentBlock[0]) {
-  for (int k = 0; k < filterValues.length; k++) {
-// filter value should be in range of max and min value i.e
-// max>filtervalue>min
-// so filter-max should be negative
-int maxCompare = 
ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMaxValue);
-// if any filter value is in range than this block needs to be
-// scanned less than equal to max range.
-if (maxCompare <= 0) {
-  isScanRequired = true;
-  break;
-}
+for (int k = 0; k < filterValues.length; k++) {
+  // filter value should be in range of max and min value i.e
+  // max>filtervalue>min
+  // so filter-max should be negative
+  int maxCompare = 
ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMaxValue);
+  // if any filter value is in range than this block needs to be
+  // scanned less than equal to max range.
+  if (maxCompare <= 0) {
+isScanRequired = true;
+break;
   }
-} else {
-  isScanRequired = isDefaultValuePresentInFilter;
 }
 return isScanRequired;
   }
 
+  private boolean isScanRequired(byte[] maxValue, byte[][] filterValue,
+  DataType dataType) {
+for (int i = 0; i < filterValue.length; i++) {
+  if (filterValue[i].length == 0 || maxValue.length == 0) {
+return isScanRequired(maxValue, filterValue);
+  }
+  switch (dataType) {
+case DOUBLE:
+  double maxValueDouble = ByteBuffer.wrap(maxValue).getDouble();
+  double filterValueDouble = 
ByteBuffer.wrap(filterValue[i]).getDouble();
+  if (filterValueDouble <= maxValueDouble) {
+return true;
+  }
+  break;
+case INT:
+case SHORT:
+case LONG:
+  long maxValueLong = ByteBuffer.wrap(maxValue).getLong();
+  long filterValueLong = ByteBuffer.wrap(filterValue[i]).getLong();
+  if (filterValueLong <= maxValueLong) {
+return true;
+  }
+  break;
+case DECIMAL:
+  BigDecimal maxDecimal = DataTypeUtil.byteToBigDecimal(maxValue);
+  BigDecimal filterDecimal = 
DataTypeUtil.byteToBigDecimal(filterValue[i]);
+  if (filterDecimal.compareTo(maxDecimal) <= 0) {
+return true;
+  }
+  }
+}
+return false;
+  }
+
   @Override public BitSetGroup applyFilter(BlocksChunkHolder 
blockChunkHolder)
   throws FilterUnsupportedException, IOException {
 // select all rows if dimension does not exists in the current block
-if (!isDimensionPresentInCurrentBlock[0]) {
+if (!isDimensionPresentInCurrentBlock[0] && 
!isMeasurePresentInCurrentBlock[0]) {
   int numberOfRows = blockChunkHolder.getDataBlock().nodeSize();
   return FilterUtil
   
.createBitSetGroupWithDefaultValue(blockChunkHolder.getDataBlock().numberOfPages(),
   numberOfRows, true);
 }
-int blockIndex =
-
segmentProperties.getDimensionOrdinalToBlockMapping().get(dimensionBlocksIndex[0]);
-if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
-  blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = 
blockChunkHolder.getDataBlock()
-  .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
-}
-DimensionRawColumnChunk rawColumnChunk =
-blockChunkHolder.getDimensionRawDataChunk()[blockIndex];
-BitSetGroup bitSetGroup = new 
BitSetGroup(rawColumnChunk.getPagesCount());
-for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
-  if (rawColumnChunk.getMaxValues() != null) {
-if (isScanRequired(rawColumnChunk.getMaxValues()[i], 
this.filterRangeValues)) {
-  int compare = ByteUtil.UnsafeComparer.INSTANCE
-  .compareTo(filterRangeValues[0], 
rawColumnChunk.getMinValues()[i]);
-  if (compare <= 0) {
-BitSet bitSet = new BitSet(rawColumnChunk.getRowCount()[i]);
-bitSet.flip(0, rawColumnChunk.getRowCount()[i]);
-bitSetGroup.setBitSet(bitSet, i);
-  } else {

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-06-30 Thread ravipesala

Github user ravipesala commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125154219
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
 ---
@@ -91,67 +131,167 @@ private void ifDefaultValueMatchesFilter() {
 
   private boolean isScanRequired(byte[] blockMaxValue, byte[][] 
filterValues) {
 boolean isScanRequired = false;
-if (isDimensionPresentInCurrentBlock[0]) {
-  for (int k = 0; k < filterValues.length; k++) {
-// filter value should be in range of max and min value i.e
-// max>filtervalue>min
-// so filter-max should be negative
-int maxCompare = 
ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMaxValue);
-// if any filter value is in range than this block needs to be
-// scanned less than equal to max range.
-if (maxCompare <= 0) {
-  isScanRequired = true;
-  break;
-}
+for (int k = 0; k < filterValues.length; k++) {
+  // filter value should be in range of max and min value i.e
+  // max>filtervalue>min
+  // so filter-max should be negative
+  int maxCompare = 
ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMaxValue);
+  // if any filter value is in range than this block needs to be
+  // scanned less than equal to max range.
+  if (maxCompare <= 0) {
+isScanRequired = true;
+break;
   }
-} else {
-  isScanRequired = isDefaultValuePresentInFilter;
 }
 return isScanRequired;
   }
 
+  private boolean isScanRequired(byte[] maxValue, byte[][] filterValue,
+  DataType dataType) {
+for (int i = 0; i < filterValue.length; i++) {
+  if (filterValue[i].length == 0 || maxValue.length == 0) {
+return isScanRequired(maxValue, filterValue);
+  }
+  switch (dataType) {
--- End diff --

Use existing methods of `DataTypeUtil` and comparator here


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-06-30 Thread ravipesala

Github user ravipesala commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125154119
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
 ---
@@ -74,80 +87,205 @@ private void ifDefaultValueMatchesFilter() {
   }
 }
   }
+} else if (!msrColEvalutorInfoList.isEmpty() && 
!isMeasurePresentInCurrentBlock[0]) {
+  CarbonMeasure measure = 
this.msrColEvalutorInfoList.get(0).getMeasure();
+  byte[] defaultValue = measure.getDefaultValue();
+  if (null != defaultValue) {
+for (int k = 0; k < filterRangeValues.length; k++) {
+  int maxCompare =
+  
ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterRangeValues[k], defaultValue);
+  if (maxCompare < 0) {
+isDefaultValuePresentInFilter = true;
+break;
+  }
+}
+  }
 }
   }
 
   @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] 
blockMinValue) {
 BitSet bitSet = new BitSet(1);
-boolean isScanRequired =
-isScanRequired(blockMaxValue[dimensionBlocksIndex[0]], 
filterRangeValues);
+boolean isScanRequired = false;
+byte[] maxValue = null;
+if (isMeasurePresentInCurrentBlock[0] || 
isDimensionPresentInCurrentBlock[0]) {
+  if (isMeasurePresentInCurrentBlock[0]) {
+maxValue = blockMaxValue[measureBlocksIndex[0] + 
lastDimensionColOrdinal];
+isScanRequired =
+isScanRequired(maxValue, filterRangeValues, 
msrColEvalutorInfoList.get(0).getType());
+  } else {
+maxValue = blockMaxValue[dimensionBlocksIndex[0]];
+isScanRequired = isScanRequired(maxValue, filterRangeValues);
+  }
+} else {
+  isScanRequired = isDefaultValuePresentInFilter;
+}
+
 if (isScanRequired) {
   bitSet.set(0);
 }
 return bitSet;
   }
 
+
   private boolean isScanRequired(byte[] blockMaxValue, byte[][] 
filterValues) {
 boolean isScanRequired = false;
-if (isDimensionPresentInCurrentBlock[0]) {
-  for (int k = 0; k < filterValues.length; k++) {
-// filter value should be in range of max and min value i.e
-// max>filtervalue>min
-// so filter-max should be negative
-int maxCompare = 
ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMaxValue);
-// if any filter value is in range than this block needs to be
-// scanned means always less than block max range.
-if (maxCompare < 0) {
-  isScanRequired = true;
-  break;
-}
+for (int k = 0; k < filterValues.length; k++) {
+  // filter value should be in range of max and min value i.e
+  // max>filtervalue>min
+  // so filter-max should be negative
+  int maxCompare = 
ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMaxValue);
+  // if any filter value is in range than this block needs to be
+  // scanned less than equal to max range.
+  if (maxCompare < 0) {
+isScanRequired = true;
+break;
   }
-} else {
-  isScanRequired = isDefaultValuePresentInFilter;
 }
 return isScanRequired;
   }
 
+  private boolean isScanRequired(byte[] maxValue, byte[][] filterValue,
+  DataType dataType) {
+for (int i = 0; i < filterValue.length; i++) {
+  if (filterValue[i].length == 0 || maxValue.length == 0) {
+return isScanRequired(maxValue, filterValue);
+  }
+  switch (dataType) {
--- End diff --

Use existing DataTypeUtil  methods and comparator here to compare


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-06-30 Thread ravipesala

Github user ravipesala commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125154090
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
 ---
@@ -74,80 +87,205 @@ private void ifDefaultValueMatchesFilter() {
   }
 }
   }
+} else if (!msrColEvalutorInfoList.isEmpty() && 
!isMeasurePresentInCurrentBlock[0]) {
+  CarbonMeasure measure = 
this.msrColEvalutorInfoList.get(0).getMeasure();
+  byte[] defaultValue = measure.getDefaultValue();
+  if (null != defaultValue) {
+for (int k = 0; k < filterRangeValues.length; k++) {
+  int maxCompare =
+  
ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterRangeValues[k], defaultValue);
--- End diff --

This comparison in case of measure is wrong. Always compare actual values. 


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-06-30 Thread ravipesala

Github user ravipesala commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125153811
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/scan/filter/executer/MeasureColumnExecuterFilterInfo.java
 ---
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.scan.filter.executer;
+
+public class MeasureColumnExecuterFilterInfo {
+
+  byte[][] filterKeys;
--- End diff --

Use Object[]


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-06-30 Thread ravipesala

Github user ravipesala commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125153772
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
 ---
@@ -186,12 +314,60 @@ private boolean isScanRequired(byte[] blkMaxVal, 
byte[] blkMinVal, byte[][] filt
 return isScanRequired;
   }
 
+  private boolean isScanRequired(byte[] maxValue, byte[] minValue, 
byte[][] filterValue,
+  DataType dataType) {
+for (int i = 0; i < filterValue.length; i++) {
+  if (filterValue[i].length == 0 || maxValue.length == 0 || 
minValue.length == 0) {
+return isScanRequired(maxValue, minValue, filterValue);
+  } else {
+switch (dataType) {
--- End diff --

Use existing methods of DataTypeUtil for conversions. And use comparator 
which is used in applyFilter method here


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-06-30 Thread ravipesala

Github user ravipesala commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125153736
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
 ---
@@ -152,12 +261,31 @@ private BitSet 
setFilterdIndexToBitSet(DimensionColumnDataChunk dimensionColumnD
 
   public BitSet isScanRequired(byte[][] blkMaxVal, byte[][] blkMinVal) {
 BitSet bitSet = new BitSet(1);
-byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
-int columnIndex = dimColumnEvaluatorInfo.getColumnIndex();
-int blockIndex = 
segmentProperties.getDimensionOrdinalToBlockMapping().get(columnIndex);
+byte[][] filterValues = null;
+int columnIndex = 0;
+int blockIndex = 0;
+boolean isScanRequired = false;
+
+if (isDimensionPresentInCurrentBlock == true) {
+  filterValues = dimColumnExecuterInfo.getFilterKeys();
+  columnIndex = dimColumnEvaluatorInfo.getColumnIndex();
+  blockIndex = 
segmentProperties.getDimensionOrdinalToBlockMapping().get(columnIndex);
+  isScanRequired =
+  isScanRequired(blkMaxVal[blockIndex], blkMinVal[blockIndex], 
filterValues);
+
+} else if (isMeasurePresentInCurrentBlock) {
+  filterValues = msrColumnExecutorInfo.getFilterKeys();
+  columnIndex = msrColumnEvaluatorInfo.getColumnIndex();
+  // blockIndex =
+  // 
segmentProperties.getDimensionOrdinalToBlockMapping().get(columnIndex) + 
segmentProperties
+  // .getLastDimensionColOrdinal();
--- End diff --

remove commented code


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-06-30 Thread ravipesala

Github user ravipesala commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125153674
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
 ---
@@ -17,65 +17,174 @@
 package org.apache.carbondata.core.scan.filter.executer;
 
 import java.io.IOException;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
 import java.util.BitSet;
+import java.util.Comparator;
 
 import org.apache.carbondata.core.datastore.block.SegmentProperties;
 import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
 import 
org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
+import 
org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
+import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.scan.filter.FilterUtil;
+import 
org.apache.carbondata.core.scan.filter.partition.PartitionFilterUtil;
 import 
org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+import 
org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
 import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
 import org.apache.carbondata.core.util.BitSetGroup;
 import org.apache.carbondata.core.util.ByteUtil;
 import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.DataTypeUtil;
 
 public class IncludeFilterExecuterImpl implements FilterExecuter {
 
   protected DimColumnResolvedFilterInfo dimColumnEvaluatorInfo;
   protected DimColumnExecuterFilterInfo dimColumnExecuterInfo;
+  protected MeasureColumnResolvedFilterInfo msrColumnEvaluatorInfo;
+  protected MeasureColumnExecuterFilterInfo msrColumnExecutorInfo;
   protected SegmentProperties segmentProperties;
+  protected boolean isDimensionPresentInCurrentBlock = false;
+  protected boolean isMeasurePresentInCurrentBlock = false;
--- End diff --

remove this flags and use null check of `dimColumnExecuterInfo` and 
`msrColumnExecutorInfo`


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-06-30 Thread ravipesala

Github user ravipesala commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125153654
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
 ---
@@ -17,65 +17,174 @@
 package org.apache.carbondata.core.scan.filter.executer;
 
 import java.io.IOException;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
 import java.util.BitSet;
+import java.util.Comparator;
 
 import org.apache.carbondata.core.datastore.block.SegmentProperties;
 import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
 import 
org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
+import 
org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
+import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.scan.filter.FilterUtil;
+import 
org.apache.carbondata.core.scan.filter.partition.PartitionFilterUtil;
 import 
org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+import 
org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
 import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
 import org.apache.carbondata.core.util.BitSetGroup;
 import org.apache.carbondata.core.util.ByteUtil;
 import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.DataTypeUtil;
 
 public class IncludeFilterExecuterImpl implements FilterExecuter {
 
   protected DimColumnResolvedFilterInfo dimColumnEvaluatorInfo;
   protected DimColumnExecuterFilterInfo dimColumnExecuterInfo;
+  protected MeasureColumnResolvedFilterInfo msrColumnEvaluatorInfo;
+  protected MeasureColumnExecuterFilterInfo msrColumnExecutorInfo;
   protected SegmentProperties segmentProperties;
+  protected boolean isDimensionPresentInCurrentBlock = false;
+  protected boolean isMeasurePresentInCurrentBlock = false;
   /**
* is dimension column data is natural sorted
*/
-  private boolean isNaturalSorted;
+  private boolean isNaturalSorted = false;
--- End diff --

default is false only right, no need to add


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-06-30 Thread ravipesala

Github user ravipesala commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125153644
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
 ---
@@ -18,56 +18,152 @@
 
 import java.io.IOException;
 import java.util.BitSet;
+import java.util.Comparator;
 
 import org.apache.carbondata.core.datastore.block.SegmentProperties;
 import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
 import 
org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
+import 
org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
+import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.scan.filter.FilterUtil;
+import 
org.apache.carbondata.core.scan.filter.partition.PartitionFilterUtil;
 import 
org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+import 
org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
 import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
 import org.apache.carbondata.core.util.BitSetGroup;
 import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.DataTypeUtil;
 
 public class ExcludeFilterExecuterImpl implements FilterExecuter {
 
   protected DimColumnResolvedFilterInfo dimColEvaluatorInfo;
   protected DimColumnExecuterFilterInfo dimColumnExecuterInfo;
+  protected MeasureColumnResolvedFilterInfo msrColumnEvaluatorInfo;
+  protected MeasureColumnExecuterFilterInfo msrColumnExecutorInfo;
   protected SegmentProperties segmentProperties;
+  protected boolean isDimensionPresentInCurrentBlock = false;
+  protected boolean isMeasurePresentInCurrentBlock = false;
   /**
* is dimension column data is natural sorted
*/
-  private boolean isNaturalSorted;
+  private boolean isNaturalSorted = false;
+
   public ExcludeFilterExecuterImpl(DimColumnResolvedFilterInfo 
dimColEvaluatorInfo,
-  SegmentProperties segmentProperties) {
-this.dimColEvaluatorInfo = dimColEvaluatorInfo;
-dimColumnExecuterInfo = new DimColumnExecuterFilterInfo();
+  MeasureColumnResolvedFilterInfo msrColumnEvaluatorInfo, 
SegmentProperties segmentProperties,
+  boolean isMeasure) {
 this.segmentProperties = segmentProperties;
-
FilterUtil.prepareKeysFromSurrogates(dimColEvaluatorInfo.getFilterValues(), 
segmentProperties,
-dimColEvaluatorInfo.getDimension(), dimColumnExecuterInfo);
-isNaturalSorted = 
dimColEvaluatorInfo.getDimension().isUseInvertedIndex() && dimColEvaluatorInfo
-.getDimension().isSortColumn();
+if (isMeasure == false) {
+  this.dimColEvaluatorInfo = dimColEvaluatorInfo;
+  dimColumnExecuterInfo = new DimColumnExecuterFilterInfo();
+
+  
FilterUtil.prepareKeysFromSurrogates(dimColEvaluatorInfo.getFilterValues(), 
segmentProperties,
+  dimColEvaluatorInfo.getDimension(), dimColumnExecuterInfo, null, 
null);
+  isDimensionPresentInCurrentBlock = true;
+  isNaturalSorted =
+  dimColEvaluatorInfo.getDimension().isUseInvertedIndex() && 
dimColEvaluatorInfo
+  .getDimension().isSortColumn();
+} else {
+  this.msrColumnEvaluatorInfo = msrColumnEvaluatorInfo;
+  msrColumnExecutorInfo = new MeasureColumnExecuterFilterInfo();
+  FilterUtil
+  
.prepareKeysFromSurrogates(msrColumnEvaluatorInfo.getFilterValues(), 
segmentProperties,
+  null, null, msrColumnEvaluatorInfo.getMeasure(), 
msrColumnExecutorInfo);
+  isMeasurePresentInCurrentBlock = true;
+}
+
   }
 
   @Override public BitSetGroup applyFilter(BlocksChunkHolder 
blockChunkHolder) throws IOException {
-int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
-.get(dimColEvaluatorInfo.getColumnIndex());
-if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
-  blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = 
blockChunkHolder.getDataBlock()
-  .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+if (isDimensionPresentInCurrentBlock == true) {
+  int blockIndex = 
segmentProperties.getDimensionOrdinalToBlockMapping()
+  .get(dimColEvaluatorInfo.getColumnIndex());
+  if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) 
{
+blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = 
blockChunkHolder.getDataBlock()
+.getDimensionChunk(blockChunkHolder.getFileReader(), 
blockIndex);

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-06-30 Thread ravipesala

Github user ravipesala commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125153317
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java ---
@@ -113,6 +115,143 @@ public static Object 
getMeasureValueBasedOnDataType(String msrValue, DataType da
 }
   }
 
+  public static Object getMeasureObjectFromDataType(byte[] data, DataType 
dataType) {
+ByteBuffer bb = ByteBuffer.wrap(data);
+switch (dataType) {
+  case SHORT:
+  case INT:
+  case LONG:
+return bb.getLong();
+  case DECIMAL:
+return byteToBigDecimal(data);
+  default:
+return bb.getDouble();
+}
+  }
+
+  /**
+   * This method will convert a given ByteArray to its specific type
+   *
+   * @param msrValue
+   * @param dataType
+   * @param carbonMeasure
+   * @return
+   */
+  //  public static byte[] getMeasureByteArrayBasedOnDataType(String 
msrValue, DataType dataType,
+  //  CarbonMeasure carbonMeasure) {
+  //switch (dataType) {
+  //  case DECIMAL:
+  //BigDecimal bigDecimal =
+  //new 
BigDecimal(msrValue).setScale(carbonMeasure.getScale(), RoundingMode.HALF_UP);
+  //   return ByteUtil.toBytes(normalizeDecimalValue(bigDecimal, 
carbonMeasure.getPrecision()));
+  //  case SHORT:
+  //return ByteUtil.toBytes((Short.parseShort(msrValue)));
+  //  case INT:
+  //return ByteUtil.toBytes(Integer.parseInt(msrValue));
+  //  case LONG:
+  //return ByteUtil.toBytes(Long.valueOf(msrValue));
+  //  default:
+  //Double parsedValue = Double.valueOf(msrValue);
+  //if (Double.isInfinite(parsedValue) || 
Double.isNaN(parsedValue)) {
+  //  return null;
+  //}
+  //return ByteUtil.toBytes(parsedValue);
+  //}
+  //  }
+  public static byte[] getMeasureByteArrayBasedOnDataTypes(String 
msrValue, DataType dataType,
+  CarbonMeasure carbonMeasure) {
+ByteBuffer b;
+switch (dataType) {
+  case BYTE:
+  case SHORT:
+  case INT:
+  case LONG:
+b = ByteBuffer.allocate(8);
+b.putLong(Long.valueOf(msrValue));
+b.flip();
+return b.array();
+  case DOUBLE:
+b = ByteBuffer.allocate(8);
+b.putDouble(Double.valueOf(msrValue));
+b.flip();
+return b.array();
+  case DECIMAL:
+BigDecimal bigDecimal =
+new BigDecimal(msrValue).setScale(carbonMeasure.getScale(), 
RoundingMode.HALF_UP);
+return DataTypeUtil
+.bigDecimalToByte(normalizeDecimalValue(bigDecimal, 
carbonMeasure.getPrecision()));
+  default:
+throw new IllegalArgumentException("Invalid data type: " + 
dataType);
+}
+  }
+
+  /**
+   * This method will convert a given ByteArray to its specific type
+   *
+   * @param msrValue
+   * @param dataType
+   * @param carbonMeasure
+   * @return
+   */
+  public static byte[] getMeasureByteArrayBasedOnDataType(ColumnPage 
measurePage, int index,
+  DataType dataType, CarbonMeasure carbonMeasure) {
+switch (dataType) {
+  case DECIMAL:
+BigDecimal bigDecimal = new 
BigDecimal(measurePage.getDouble(index))
+.setScale(carbonMeasure.getScale(), RoundingMode.HALF_UP);
+return ByteUtil.toBytes(normalizeDecimalValue(bigDecimal, 
carbonMeasure.getPrecision()));
+  case SHORT:
+return ByteUtil.toBytes(measurePage.getShort(index));
+  case INT:
+return ByteUtil.toBytes(measurePage.getInt(index));
+  case LONG:
+return ByteUtil.toBytes(measurePage.getLong(index));
+  default:
+Double parsedValue = Double.valueOf(measurePage.getDouble(index));
+if (Double.isInfinite(parsedValue) || Double.isNaN(parsedValue)) {
+  return null;
+}
+return ByteUtil.toBytes(parsedValue);
+}
+  }
+
+  public static Object getMeasureObjectBasedOnDataType(ColumnPage 
measurePage, int index,
+  DataType dataType, CarbonMeasure carbonMeasure) {
+//switch (dataType) {
+//  case DECIMAL:
+//BigDecimal bigDecimal = new 
BigDecimal(measurePage.getDouble(index))
+//.setScale(carbonMeasure.getScale(), 
RoundingMode.HALF_UP);
+//return normalizeDecimalValue(bigDecimal, 
carbonMeasure.getPrecision());
+//  case SHORT:
+//  case INT:
+//  case LONG:
+//return measurePage.getLong(index);
+//  default:
+//Double

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-06-30 Thread ravipesala

Github user ravipesala commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125153305
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java ---
@@ -113,6 +115,143 @@ public static Object 
getMeasureValueBasedOnDataType(String msrValue, DataType da
 }
   }
 
+  public static Object getMeasureObjectFromDataType(byte[] data, DataType 
dataType) {
+ByteBuffer bb = ByteBuffer.wrap(data);
+switch (dataType) {
+  case SHORT:
+  case INT:
+  case LONG:
+return bb.getLong();
+  case DECIMAL:
+return byteToBigDecimal(data);
+  default:
+return bb.getDouble();
+}
+  }
+
+  /**
+   * This method will convert a given ByteArray to its specific type
+   *
+   * @param msrValue
+   * @param dataType
+   * @param carbonMeasure
+   * @return
+   */
+  //  public static byte[] getMeasureByteArrayBasedOnDataType(String 
msrValue, DataType dataType,
+  //  CarbonMeasure carbonMeasure) {
+  //switch (dataType) {
+  //  case DECIMAL:
+  //BigDecimal bigDecimal =
+  //new 
BigDecimal(msrValue).setScale(carbonMeasure.getScale(), RoundingMode.HALF_UP);
+  //   return ByteUtil.toBytes(normalizeDecimalValue(bigDecimal, 
carbonMeasure.getPrecision()));
+  //  case SHORT:
+  //return ByteUtil.toBytes((Short.parseShort(msrValue)));
+  //  case INT:
+  //return ByteUtil.toBytes(Integer.parseInt(msrValue));
+  //  case LONG:
+  //return ByteUtil.toBytes(Long.valueOf(msrValue));
+  //  default:
+  //Double parsedValue = Double.valueOf(msrValue);
+  //if (Double.isInfinite(parsedValue) || 
Double.isNaN(parsedValue)) {
+  //  return null;
+  //}
+  //return ByteUtil.toBytes(parsedValue);
+  //}
+  //  }
+  public static byte[] getMeasureByteArrayBasedOnDataTypes(String 
msrValue, DataType dataType,
+  CarbonMeasure carbonMeasure) {
+ByteBuffer b;
+switch (dataType) {
+  case BYTE:
+  case SHORT:
+  case INT:
+  case LONG:
+b = ByteBuffer.allocate(8);
+b.putLong(Long.valueOf(msrValue));
+b.flip();
+return b.array();
+  case DOUBLE:
+b = ByteBuffer.allocate(8);
+b.putDouble(Double.valueOf(msrValue));
+b.flip();
+return b.array();
+  case DECIMAL:
+BigDecimal bigDecimal =
+new BigDecimal(msrValue).setScale(carbonMeasure.getScale(), 
RoundingMode.HALF_UP);
+return DataTypeUtil
+.bigDecimalToByte(normalizeDecimalValue(bigDecimal, 
carbonMeasure.getPrecision()));
+  default:
+throw new IllegalArgumentException("Invalid data type: " + 
dataType);
+}
+  }
+
+  /**
+   * This method will convert a given ByteArray to its specific type
+   *
+   * @param msrValue
+   * @param dataType
+   * @param carbonMeasure
+   * @return
+   */
+  public static byte[] getMeasureByteArrayBasedOnDataType(ColumnPage 
measurePage, int index,
--- End diff --

This method is not used, please remove it


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-06-30 Thread ravipesala

Github user ravipesala commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125153193
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java ---
@@ -113,6 +115,143 @@ public static Object 
getMeasureValueBasedOnDataType(String msrValue, DataType da
 }
   }
 
+  public static Object getMeasureObjectFromDataType(byte[] data, DataType 
dataType) {
+ByteBuffer bb = ByteBuffer.wrap(data);
+switch (dataType) {
+  case SHORT:
+  case INT:
+  case LONG:
+return bb.getLong();
+  case DECIMAL:
+return byteToBigDecimal(data);
+  default:
+return bb.getDouble();
+}
+  }
+
+  /**
+   * This method will convert a given ByteArray to its specific type
+   *
+   * @param msrValue
+   * @param dataType
+   * @param carbonMeasure
+   * @return
+   */
+  //  public static byte[] getMeasureByteArrayBasedOnDataType(String 
msrValue, DataType dataType,
+  //  CarbonMeasure carbonMeasure) {
+  //switch (dataType) {
+  //  case DECIMAL:
+  //BigDecimal bigDecimal =
+  //new 
BigDecimal(msrValue).setScale(carbonMeasure.getScale(), RoundingMode.HALF_UP);
+  //   return ByteUtil.toBytes(normalizeDecimalValue(bigDecimal, 
carbonMeasure.getPrecision()));
+  //  case SHORT:
+  //return ByteUtil.toBytes((Short.parseShort(msrValue)));
+  //  case INT:
+  //return ByteUtil.toBytes(Integer.parseInt(msrValue));
+  //  case LONG:
+  //return ByteUtil.toBytes(Long.valueOf(msrValue));
+  //  default:
+  //Double parsedValue = Double.valueOf(msrValue);
+  //if (Double.isInfinite(parsedValue) || 
Double.isNaN(parsedValue)) {
+  //  return null;
+  //}
+  //return ByteUtil.toBytes(parsedValue);
+  //}
+  //  }
--- End diff --

remove the commented code


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-06-30 Thread ravipesala

Github user ravipesala commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125153179
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java ---
@@ -113,6 +115,143 @@ public static Object 
getMeasureValueBasedOnDataType(String msrValue, DataType da
 }
   }
 
+  public static Object getMeasureObjectFromDataType(byte[] data, DataType 
dataType) {
+ByteBuffer bb = ByteBuffer.wrap(data);
--- End diff --

This is unnecessary object for `decimal` so keep it in respective case 
statement


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-06-30 Thread ravipesala

Github user ravipesala commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125153056
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
 ---
@@ -18,56 +18,152 @@
 
 import java.io.IOException;
 import java.util.BitSet;
+import java.util.Comparator;
 
 import org.apache.carbondata.core.datastore.block.SegmentProperties;
 import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
 import 
org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
+import 
org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
+import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.scan.filter.FilterUtil;
+import 
org.apache.carbondata.core.scan.filter.partition.PartitionFilterUtil;
 import 
org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+import 
org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
 import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
 import org.apache.carbondata.core.util.BitSetGroup;
 import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.DataTypeUtil;
 
 public class ExcludeFilterExecuterImpl implements FilterExecuter {
 
   protected DimColumnResolvedFilterInfo dimColEvaluatorInfo;
   protected DimColumnExecuterFilterInfo dimColumnExecuterInfo;
+  protected MeasureColumnResolvedFilterInfo msrColumnEvaluatorInfo;
+  protected MeasureColumnExecuterFilterInfo msrColumnExecutorInfo;
   protected SegmentProperties segmentProperties;
+  protected boolean isDimensionPresentInCurrentBlock = false;
+  protected boolean isMeasurePresentInCurrentBlock = false;
--- End diff --

I don't think all these flags are required. just do `null` check of  
`dimColumnExecuterInfo` and  `msrColumnExecutorInfo` is enough


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-06-30 Thread ravipesala

Github user ravipesala commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125152979
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
 ---
@@ -18,56 +18,152 @@
 
 import java.io.IOException;
 import java.util.BitSet;
+import java.util.Comparator;
 
 import org.apache.carbondata.core.datastore.block.SegmentProperties;
 import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
 import 
org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
+import 
org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
+import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.scan.filter.FilterUtil;
+import 
org.apache.carbondata.core.scan.filter.partition.PartitionFilterUtil;
 import 
org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+import 
org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
 import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
 import org.apache.carbondata.core.util.BitSetGroup;
 import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.DataTypeUtil;
 
 public class ExcludeFilterExecuterImpl implements FilterExecuter {
 
   protected DimColumnResolvedFilterInfo dimColEvaluatorInfo;
   protected DimColumnExecuterFilterInfo dimColumnExecuterInfo;
+  protected MeasureColumnResolvedFilterInfo msrColumnEvaluatorInfo;
+  protected MeasureColumnExecuterFilterInfo msrColumnExecutorInfo;
   protected SegmentProperties segmentProperties;
+  protected boolean isDimensionPresentInCurrentBlock = false;
+  protected boolean isMeasurePresentInCurrentBlock = false;
   /**
* is dimension column data is natural sorted
*/
-  private boolean isNaturalSorted;
+  private boolean isNaturalSorted = false;
+
   public ExcludeFilterExecuterImpl(DimColumnResolvedFilterInfo 
dimColEvaluatorInfo,
-  SegmentProperties segmentProperties) {
-this.dimColEvaluatorInfo = dimColEvaluatorInfo;
-dimColumnExecuterInfo = new DimColumnExecuterFilterInfo();
+  MeasureColumnResolvedFilterInfo msrColumnEvaluatorInfo, 
SegmentProperties segmentProperties,
+  boolean isMeasure) {
 this.segmentProperties = segmentProperties;
-
FilterUtil.prepareKeysFromSurrogates(dimColEvaluatorInfo.getFilterValues(), 
segmentProperties,
-dimColEvaluatorInfo.getDimension(), dimColumnExecuterInfo);
-isNaturalSorted = 
dimColEvaluatorInfo.getDimension().isUseInvertedIndex() && dimColEvaluatorInfo
-.getDimension().isSortColumn();
+if (isMeasure == false) {
--- End diff --

just use `!isMeasure`


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-06-30 Thread ravipesala

Github user ravipesala commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125152956
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java ---
@@ -1042,12 +1144,17 @@ public static FilterExecuter getFilterExecuterTree(
* @param dimension
* @param dimColumnExecuterInfo
*/
-  public static void prepareKeysFromSurrogates(DimColumnFilterInfo 
filterValues,
+  public static void prepareKeysFromSurrogates(ColumnFilterInfo 
filterValues,
   SegmentProperties segmentProperties, CarbonDimension dimension,
-  DimColumnExecuterFilterInfo dimColumnExecuterInfo) {
-byte[][] keysBasedOnFilter = getKeyArray(filterValues, dimension, 
segmentProperties);
-dimColumnExecuterInfo.setFilterKeys(keysBasedOnFilter);
-
+  DimColumnExecuterFilterInfo dimColumnExecuterInfo, CarbonMeasure 
measures,
+  MeasureColumnExecuterFilterInfo msrColumnExecuterInfo) {
+if (null != measures) {
--- End diff --

I don't think this `if ` check is required. just pass dimension and measure 
to the method 'getKeyArray'


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-06-30 Thread ravipesala

Github user ravipesala commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125152881
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java ---
@@ -395,6 +440,58 @@ public static DimColumnFilterInfo 
getNoDictionaryValKeyMemberForFilter(
   }
 
   /**
+   * This method will get the no dictionary data based on filters and same
+   * will be in ColumnFilterInfo
+   *
+   * @param evaluateResultListFinal
+   * @param isIncludeFilter
+   * @return ColumnFilterInfo
+   */
+  public static ColumnFilterInfo getMeasureValKeyMemberForFilter(
+  List evaluateResultListFinal, boolean isIncludeFilter, 
DataType dataType,
+  CarbonMeasure carbonMeasure) throws FilterUnsupportedException {
+List filterValuesList = new ArrayList(20);
+String result = null;
+try {
+  int length = evaluateResultListFinal.size();
+  for (int i = 0; i < length; i++) {
+result = evaluateResultListFinal.get(i);
+if (CarbonCommonConstants.MEMBER_DEFAULT_VAL.equals(result)) {
+  filterValuesList.add(new byte[0]);
+  continue;
+}
+// TODO have to understand what method to be used for measures.
+// filterValuesList
+//  
.add(DataTypeUtil.getBytesBasedOnDataTypeForNoDictionaryColumn(result, 
dataType));
+
+filterValuesList
+.add(DataTypeUtil.getMeasureByteArrayBasedOnDataTypes(result, 
dataType, carbonMeasure));
+
+  }
+} catch (Throwable ex) {
+  throw new FilterUnsupportedException("Unsupported Filter condition: 
" + result, ex);
+}
+
+Comparator filterMeasureComaparator = new Comparator() 
{
+
+  @Override public int compare(byte[] filterMember1, byte[] 
filterMember2) {
+// TODO Auto-generated method stub
+return ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterMember1, 
filterMember2);
--- End diff --

Please compare actual values before convert to binary


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-06-30 Thread ravipesala

Github user ravipesala commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125152873
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java ---
@@ -395,6 +440,58 @@ public static DimColumnFilterInfo 
getNoDictionaryValKeyMemberForFilter(
   }
 
   /**
+   * This method will get the no dictionary data based on filters and same
+   * will be in ColumnFilterInfo
+   *
+   * @param evaluateResultListFinal
+   * @param isIncludeFilter
+   * @return ColumnFilterInfo
+   */
+  public static ColumnFilterInfo getMeasureValKeyMemberForFilter(
+  List evaluateResultListFinal, boolean isIncludeFilter, 
DataType dataType,
+  CarbonMeasure carbonMeasure) throws FilterUnsupportedException {
+List filterValuesList = new ArrayList(20);
+String result = null;
+try {
+  int length = evaluateResultListFinal.size();
+  for (int i = 0; i < length; i++) {
+result = evaluateResultListFinal.get(i);
+if (CarbonCommonConstants.MEMBER_DEFAULT_VAL.equals(result)) {
+  filterValuesList.add(new byte[0]);
+  continue;
+}
+// TODO have to understand what method to be used for measures.
+// filterValuesList
+//  
.add(DataTypeUtil.getBytesBasedOnDataTypeForNoDictionaryColumn(result, 
dataType));
+
+filterValuesList
+.add(DataTypeUtil.getMeasureByteArrayBasedOnDataTypes(result, 
dataType, carbonMeasure));
+
+  }
+} catch (Throwable ex) {
+  throw new FilterUnsupportedException("Unsupported Filter condition: 
" + result, ex);
+}
+
+Comparator filterMeasureComaparator = new Comparator() 
{
+
+  @Override public int compare(byte[] filterMember1, byte[] 
filterMember2) {
+// TODO Auto-generated method stub
+return ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterMember1, 
filterMember2);
--- End diff --

this is wrong, we cannot compare `double,float,decimal` through bytes. 


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-06-30 Thread ravipesala

Github user ravipesala commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125152824
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java ---
@@ -395,6 +440,58 @@ public static DimColumnFilterInfo 
getNoDictionaryValKeyMemberForFilter(
   }
 
   /**
+   * This method will get the no dictionary data based on filters and same
+   * will be in ColumnFilterInfo
+   *
+   * @param evaluateResultListFinal
+   * @param isIncludeFilter
+   * @return ColumnFilterInfo
+   */
+  public static ColumnFilterInfo getMeasureValKeyMemberForFilter(
+  List evaluateResultListFinal, boolean isIncludeFilter, 
DataType dataType,
+  CarbonMeasure carbonMeasure) throws FilterUnsupportedException {
+List filterValuesList = new ArrayList(20);
+String result = null;
+try {
+  int length = evaluateResultListFinal.size();
+  for (int i = 0; i < length; i++) {
+result = evaluateResultListFinal.get(i);
+if (CarbonCommonConstants.MEMBER_DEFAULT_VAL.equals(result)) {
+  filterValuesList.add(new byte[0]);
+  continue;
+}
+// TODO have to understand what method to be used for measures.
+// filterValuesList
+//  
.add(DataTypeUtil.getBytesBasedOnDataTypeForNoDictionaryColumn(result, 
dataType));
--- End diff --

is this comment required now? please remove if not required


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-06-30 Thread ravipesala

Github user ravipesala commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125152709
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java ---
@@ -209,9 +233,29 @@ private static FilterExecuter getIncludeFilterExecuter(
* @return
*/
   private static FilterExecuter getExcludeFilterExecuter(
-  DimColumnResolvedFilterInfo dimColResolvedFilterInfo, 
SegmentProperties segmentProperties) {
+  DimColumnResolvedFilterInfo dimColResolvedFilterInfo,
+  MeasureColumnResolvedFilterInfo msrColResolvedFilterInfo,
+  SegmentProperties segmentProperties) {
 
-if (dimColResolvedFilterInfo.getDimension().isColumnar()) {
+if (null != msrColResolvedFilterInfo && 
msrColResolvedFilterInfo.getMeasure().isColumnar()) {
--- End diff --

even here `isColumnar` check not required


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-06-30 Thread ravipesala

Github user ravipesala commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125152663
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java ---
@@ -180,9 +185,27 @@ private static FilterExecuter createFilterExecuterTree(
* @return
*/
   private static FilterExecuter getIncludeFilterExecuter(
-  DimColumnResolvedFilterInfo dimColResolvedFilterInfo, 
SegmentProperties segmentProperties) {
-
-if (dimColResolvedFilterInfo.getDimension().isColumnar()) {
+  DimColumnResolvedFilterInfo dimColResolvedFilterInfo,
+  MeasureColumnResolvedFilterInfo msrColResolvedFilterInfo,
+  SegmentProperties segmentProperties) {
+if (null != msrColResolvedFilterInfo && 
msrColResolvedFilterInfo.getMeasure().isColumnar()) {
+  CarbonMeasure measuresFromCurrentBlock = segmentProperties
+  
.getMeasureFromCurrentBlock(msrColResolvedFilterInfo.getMeasure().getColumnId());
+  if (null != measuresFromCurrentBlock) {
+// update dimension and column index according to the dimension 
position in current block
+MeasureColumnResolvedFilterInfo msrColResolvedFilterInfoCopyObject 
=
+msrColResolvedFilterInfo.getCopyObject();
+
msrColResolvedFilterInfoCopyObject.setMeasure(measuresFromCurrentBlock);
+
msrColResolvedFilterInfoCopyObject.setColumnIndex(measuresFromCurrentBlock.getOrdinal());
+
msrColResolvedFilterInfoCopyObject.setType(measuresFromCurrentBlock.getDataType());
+return new IncludeFilterExecuterImpl(null, 
msrColResolvedFilterInfoCopyObject,
+segmentProperties, true);
+  } else {
+return new 
RestructureIncludeFilterExecutorImpl(dimColResolvedFilterInfo,
+msrColResolvedFilterInfo, segmentProperties, true);
+  }
+}
--- End diff --

isn't `else` should be here for `dimColResolvedFilterInfo` check


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-06-30 Thread ravipesala

Github user ravipesala commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125152630
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java ---
@@ -180,9 +185,27 @@ private static FilterExecuter createFilterExecuterTree(
* @return
*/
   private static FilterExecuter getIncludeFilterExecuter(
-  DimColumnResolvedFilterInfo dimColResolvedFilterInfo, 
SegmentProperties segmentProperties) {
-
-if (dimColResolvedFilterInfo.getDimension().isColumnar()) {
+  DimColumnResolvedFilterInfo dimColResolvedFilterInfo,
+  MeasureColumnResolvedFilterInfo msrColResolvedFilterInfo,
+  SegmentProperties segmentProperties) {
+if (null != msrColResolvedFilterInfo && 
msrColResolvedFilterInfo.getMeasure().isColumnar()) {
--- End diff --

I don't think `msrColResolvedFilterInfo.getMeasure().isColumnar()` is 
really required. it is only for dimensions.please remove


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-06-30 Thread ravipesala

Github user ravipesala commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125152433
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/scan/expression/ColumnExpression.java
 ---
@@ -31,12 +32,16 @@
 
   private boolean isDimension;
 
+  private boolean isMeasure;
--- End diff --

same information is available inside `measure`, so please remove it and get 
it from `measure`


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-06-30 Thread ravipesala

Github user ravipesala commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125152368
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
 ---
@@ -137,6 +137,8 @@
*/
   private int numberOfNoDictSortColumns;
 
+  private int lastDimensionColumnOrdinal;
--- End diff --

Not used any where, please remove


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-06-30 Thread ravipesala

Github user ravipesala commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125152345
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/ColumnPageStatsVO.java
 ---
@@ -102,9 +109,15 @@ public void update(Object value) {
 break;
   case DECIMAL:
 BigDecimal decimalValue = DataTypeUtil.byteToBigDecimal((byte[]) 
value);
-decimal = decimalValue.scale();
-BigDecimal val = (BigDecimal) min;
-nonExistValue = (val.subtract(new BigDecimal(1.0)));
+if (isFirst) {
--- End diff --

I think this `isFirst` is required. just check null for max or min here.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-06-30 Thread ravipesala

Github user ravipesala commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/1079#discussion_r125152265
  
--- Diff: 
core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/ColumnPageStatsVO.java
 ---
@@ -56,9 +65,7 @@ public ColumnPageStatsVO(DataType dataType) {
 nonExistValue = Double.MIN_VALUE;
 break;
   case DECIMAL:
-max = new BigDecimal(Double.MIN_VALUE);
-min = new BigDecimal(Double.MAX_VALUE);
-nonExistValue = new BigDecimal(Double.MIN_VALUE);
+this.zeroDecimal = new BigDecimal(0);
--- End diff --

use `BigDecimal.ZERO`


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] carbondata pull request #1079: [WIP]Measure Filter implementation

2017-06-22 Thread sounakr

GitHub user sounakr opened a pull request:

https://github.com/apache/carbondata/pull/1079

[WIP]Measure Filter implementation

Measure Filter Implementation

You can merge this pull request into a Git repository by running:

$ git pull https://github.com/sounakr/incubator-carbondata measure_filter

Alternatively you can review and apply these changes as the patch at:

https://github.com/apache/carbondata/pull/1079.patch

To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:

This closes #1079


commit b3fa1780ae0e26fa379d812f9aec1c1c6274b8c6
Author: sounakr 
Date:   2017-06-20T17:22:36Z

Measure Filter implementation




---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

57 matches

Mail list logo