http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/keygenerator/columnar/impl/MultiDimKeyVarLengthVariableSplitGenerator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/columnar/impl/MultiDimKeyVarLengthVariableSplitGenerator.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/columnar/impl/MultiDimKeyVarLengthVariableSplitGenerator.java index cb52045..1ed4577 100644 --- a/core/src/main/java/org/apache/carbondata/core/keygenerator/columnar/impl/MultiDimKeyVarLengthVariableSplitGenerator.java +++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/columnar/impl/MultiDimKeyVarLengthVariableSplitGenerator.java @@ -110,29 +110,6 @@ public class MultiDimKeyVarLengthVariableSplitGenerator extends MultiDimKeyVarLe dimBlockArray[j] = convertToArray(dimBlockSet[j]); } - int[][] splitDimArrayLocalIndexes = new int[splitDimArray.length][]; - for (int j = 0; j < splitDimArrayLocalIndexes.length; j++) { - splitDimArrayLocalIndexes[j] = splitDimArray[j].length > 0 ? - new int[] { 0, splitDimArray[j][1] - splitDimArray[j][0] } : - new int[0]; - } - - int[][][] byteRangesForDims = new int[byteRangesForKeys.length][][]; - for (int j = 0; j < byteRangesForKeys.length; j++) { - if (dimBlockArray[j].length > 1) { - int[] bArray1 = splitDimArrayLocalIndexes[dimBlockArray[j][0]]; - byteRangesForDims[j] = new int[2][2]; - byteRangesForDims[j][0] = - new int[] { bArray1[bArray1.length - 1], bArray1[bArray1.length - 1] }; - byteRangesForDims[j][1] = new int[] { 0, - (byteRangesForKeys[j][byteRangesForKeys[j].length - 1] - byteRangesForKeys[j][0]) - 1 }; - } else { - byteRangesForDims[j] = new int[1][1]; - int[] bArray1 = splitDimArray[dimBlockArray[j][0]]; - byteRangesForDims[j][0] = new int[] { byteRangesForKeys[j][0] - bArray1[0], - byteRangesForKeys[j][1] - bArray1[0] }; - } - } blockKeySize = new int[splitDimArray.length]; for (int j = 0; j < blockKeySize.length; j++) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java index 9fc2c25..0d7cb6c 100644 --- a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java +++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java @@ -139,7 +139,7 @@ public class DateDirectDictionaryGenerator implements DirectDictionaryGenerator private int generateDirectSurrogateKeyForNonTimestampType(String memberStr) { long timeValue = -1; try { - timeValue = Long.valueOf(memberStr) / 1000; + timeValue = Long.parseLong(memberStr) / 1000; } catch (NumberFormatException e) { if (LOGGER.isDebugEnabled()) { LOGGER.debug( http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java index 5c7ba07..6a0b9e6 100644 --- a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java +++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java @@ -199,7 +199,7 @@ public class TimeStampDirectDictionaryGenerator implements DirectDictionaryGener private int generateDirectSurrogateKeyForNonTimestampType(String memberStr) { long timeValue = -1; try { - timeValue = Long.valueOf(memberStr) / 1000; + timeValue = Long.parseLong(memberStr) / 1000; } catch (NumberFormatException e) { if (LOGGER.isDebugEnabled()) { LOGGER.debug( http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java b/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java index aa24e33..752c70d 100644 --- a/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java +++ b/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java @@ -45,7 +45,7 @@ public class HdfsFileLock extends AbstractCarbonLock { private DataOutputStream dataOutputStream; - public static String tmpPath; + private static String tmpPath; static { Configuration conf = new Configuration(true); http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/locks/ZooKeeperLocking.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/locks/ZooKeeperLocking.java b/core/src/main/java/org/apache/carbondata/core/locks/ZooKeeperLocking.java index 24bc5b0..5e83aa4 100644 --- a/core/src/main/java/org/apache/carbondata/core/locks/ZooKeeperLocking.java +++ b/core/src/main/java/org/apache/carbondata/core/locks/ZooKeeperLocking.java @@ -74,6 +74,14 @@ public class ZooKeeperLocking extends AbstractCarbonLock { lockFile); } + public static void initialize() { + String zooKeeperUrl = + CarbonProperties.getInstance().getProperty(CarbonCommonConstants.ZOOKEEPER_URL); + if (null == zk) { + zk = ZookeeperInit.getInstance(zooKeeperUrl).getZookeeper(); + } + } + /** * @param lockLocation * @param lockFile @@ -82,9 +90,7 @@ public class ZooKeeperLocking extends AbstractCarbonLock { this.lockName = lockFile; this.tableIdFolder = zooKeeperLocation + CarbonCommonConstants.FILE_SEPARATOR + lockLocation; - String zooKeeperUrl = - CarbonProperties.getInstance().getProperty(CarbonCommonConstants.ZOOKEEPER_URL); - zk = ZookeeperInit.getInstance(zooKeeperUrl).getZookeeper(); + initialize(); this.lockTypeFolder = zooKeeperLocation + CarbonCommonConstants.FILE_SEPARATOR + lockLocation + CarbonCommonConstants.FILE_SEPARATOR + lockFile; http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/memory/CarbonUnsafe.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/memory/CarbonUnsafe.java b/core/src/main/java/org/apache/carbondata/core/memory/CarbonUnsafe.java index 88e108c..74343f5 100644 --- a/core/src/main/java/org/apache/carbondata/core/memory/CarbonUnsafe.java +++ b/core/src/main/java/org/apache/carbondata/core/memory/CarbonUnsafe.java @@ -40,7 +40,7 @@ public final class CarbonUnsafe { public static final boolean ISLITTLEENDIAN = ByteOrder.nativeOrder().equals(ByteOrder.LITTLE_ENDIAN); - public static Unsafe unsafe; + private static Unsafe unsafe; static { try { @@ -66,4 +66,8 @@ public final class CarbonUnsafe { DOUBLE_ARRAY_OFFSET = 0; } } + + public static Unsafe getUnsafe() { + return unsafe; + } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/memory/IntPointerBuffer.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/memory/IntPointerBuffer.java b/core/src/main/java/org/apache/carbondata/core/memory/IntPointerBuffer.java index dadb1e4..5cd8e8d 100644 --- a/core/src/main/java/org/apache/carbondata/core/memory/IntPointerBuffer.java +++ b/core/src/main/java/org/apache/carbondata/core/memory/IntPointerBuffer.java @@ -67,7 +67,7 @@ public class IntPointerBuffer { assert rowId >= 0 : "rowId (" + rowId + ") should >= 0"; assert rowId < length : "rowId (" + rowId + ") should < length (" + length + ")"; if (pointerBlock == null) { - return CarbonUnsafe.unsafe.getInt(pointerMemoryBlock.getBaseObject(), + return CarbonUnsafe.getUnsafe().getInt(pointerMemoryBlock.getBaseObject(), pointerMemoryBlock.getBaseOffset() + (rowId << 2)); } return pointerBlock[rowId]; @@ -78,7 +78,7 @@ public class IntPointerBuffer { pointerMemoryBlock = UnsafeSortMemoryManager.allocateMemoryWithRetry(this.taskId, pointerBlock.length * 4); for (int i = 0; i < pointerBlock.length; i++) { - CarbonUnsafe.unsafe + CarbonUnsafe.getUnsafe() .putInt(pointerMemoryBlock.getBaseObject(), pointerMemoryBlock.getBaseOffset() + i * 4, pointerBlock[i]); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/memory/MemoryBlock.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/memory/MemoryBlock.java b/core/src/main/java/org/apache/carbondata/core/memory/MemoryBlock.java index ab9b3d4..74136c1 100644 --- a/core/src/main/java/org/apache/carbondata/core/memory/MemoryBlock.java +++ b/core/src/main/java/org/apache/carbondata/core/memory/MemoryBlock.java @@ -29,13 +29,6 @@ public class MemoryBlock extends MemoryLocation { private final long length; - /** - * Optional page number; used when this MemoryBlock represents a page allocated by a - * TaskMemoryManager. This field is public so that it can be modified by the TaskMemoryManager, - * which lives in a different package. - */ - public int pageNumber = -1; - public MemoryBlock(@Nullable Object obj, long offset, long length) { super(obj, offset); this.length = length; http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryAllocator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryAllocator.java b/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryAllocator.java index db38ff2..fcb0b88 100644 --- a/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryAllocator.java +++ b/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryAllocator.java @@ -25,9 +25,9 @@ public class UnsafeMemoryAllocator implements MemoryAllocator { @Override public MemoryBlock allocate(long size) throws OutOfMemoryError { - long address = CarbonUnsafe.unsafe.allocateMemory(size); + long address = CarbonUnsafe.getUnsafe().allocateMemory(size); // initializing memory with zero - CarbonUnsafe.unsafe.setMemory(null, address, size, (byte) 0); + CarbonUnsafe.getUnsafe().setMemory(null, address, size, (byte) 0); return new MemoryBlock(null, address, size); } @@ -35,6 +35,6 @@ public class UnsafeMemoryAllocator implements MemoryAllocator { public void free(MemoryBlock memory) { assert (memory.obj == null) : "baseObject not null; are you trying to use the off-heap allocator to free on-heap memory?"; - CarbonUnsafe.unsafe.freeMemory(memory.offset); + CarbonUnsafe.getUnsafe().freeMemory(memory.offset); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java b/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java index d433b5e..1190d56 100644 --- a/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java +++ b/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java @@ -114,11 +114,9 @@ public class UnsafeMemoryManager { - memoryUsed)); } - public void freeMemoryAll(long taskId) { + public synchronized void freeMemoryAll(long taskId) { Set<MemoryBlock> memoryBlockSet = null; - synchronized (INSTANCE) { - memoryBlockSet = taskIdToMemoryBlockMap.remove(taskId); - } + memoryBlockSet = taskIdToMemoryBlockMap.remove(taskId); long occuppiedMemory = 0; if (null != memoryBlockSet) { Iterator<MemoryBlock> iterator = memoryBlockSet.iterator(); @@ -129,10 +127,8 @@ public class UnsafeMemoryManager { allocator.free(memoryBlock); } } - synchronized (INSTANCE) { - memoryUsed -= occuppiedMemory; - memoryUsed = memoryUsed < 0 ? 0 : memoryUsed; - } + memoryUsed -= occuppiedMemory; + memoryUsed = memoryUsed < 0 ? 0 : memoryUsed; if (LOGGER.isDebugEnabled()) { LOGGER.debug( "Freeing memory of size: " + occuppiedMemory + ": Current available memory is: " + ( http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/memory/UnsafeSortMemoryManager.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/memory/UnsafeSortMemoryManager.java b/core/src/main/java/org/apache/carbondata/core/memory/UnsafeSortMemoryManager.java index d975cd4..b4a200a 100644 --- a/core/src/main/java/org/apache/carbondata/core/memory/UnsafeSortMemoryManager.java +++ b/core/src/main/java/org/apache/carbondata/core/memory/UnsafeSortMemoryManager.java @@ -152,11 +152,9 @@ public class UnsafeSortMemoryManager { * when in case of task failure we need to clear all the memory occupied * @param taskId */ - public void freeMemoryAll(long taskId) { + public synchronized void freeMemoryAll(long taskId) { Set<MemoryBlock> memoryBlockSet = null; - synchronized (INSTANCE) { - memoryBlockSet = taskIdToMemoryBlockMap.remove(taskId); - } + memoryBlockSet = taskIdToMemoryBlockMap.remove(taskId); long occuppiedMemory = 0; if (null != memoryBlockSet) { Iterator<MemoryBlock> iterator = memoryBlockSet.iterator(); @@ -167,10 +165,8 @@ public class UnsafeSortMemoryManager { allocator.free(memoryBlock); } } - synchronized (INSTANCE) { - memoryUsed -= occuppiedMemory; - memoryUsed = memoryUsed < 0 ? 0 : memoryUsed; - } + memoryUsed -= occuppiedMemory; + memoryUsed = memoryUsed < 0 ? 0 : memoryUsed; if (LOGGER.isDebugEnabled()) { LOGGER.debug( "Freeing memory of size: " + occuppiedMemory + ": Current available memory is: " + ( http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/metadata/ColumnPageCodecMeta.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/ColumnPageCodecMeta.java b/core/src/main/java/org/apache/carbondata/core/metadata/ColumnPageCodecMeta.java index 591899e..54bc2ce 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/ColumnPageCodecMeta.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/ColumnPageCodecMeta.java @@ -194,6 +194,8 @@ public class ColumnPageCodecMeta extends ValueEncoderMeta implements Serializabl buffer.putInt(getScale()); buffer.putInt(getPrecision()); break; + default: + throw new RuntimeException("Unexpected type: " + srcDataType); } buffer.putInt(getDecimal()); buffer.put(getDataTypeSelected()); http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java index 459eb24..0343e38 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java @@ -69,11 +69,9 @@ public final class DecimalConverterFactory { private ByteBuffer buffer = ByteBuffer.allocate(4); - private int precision; private int scale; public DecimalIntConverter(int precision, int scale) { - this.precision = precision; this.scale = scale; } @@ -116,11 +114,9 @@ public final class DecimalConverterFactory { private ByteBuffer buffer = ByteBuffer.allocate(8); - private int precision; private int scale; public DecimalLongConverter(int precision, int scale) { - this.precision = precision; this.scale = scale; } @@ -147,7 +143,6 @@ public final class DecimalConverterFactory { public class DecimalUnscaledConverter implements DecimalConverter { - private int precision; private int scale; @@ -156,7 +151,6 @@ public final class DecimalConverterFactory { private byte[] decimalBuffer = new byte[minBytesForPrecision[38]]; public DecimalUnscaledConverter(int precision, int scale) { - this.precision = precision; this.scale = scale; this.numBytes = minBytesForPrecision[precision]; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java index 2bd3198..0c807f6 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java @@ -185,15 +185,15 @@ public class TableInfo implements Serializable, Writable { return false; } TableInfo other = (TableInfo) obj; - if ((databaseName == null && other.databaseName != null) || (databaseName != null - && other.databaseName == null)) { + if (null == databaseName || null == other.databaseName) { return false; } - if ((tableUniqueName == null && other.tableUniqueName != null) || (tableUniqueName != null - && other.tableUniqueName == null)) { + if (null == tableUniqueName || null == other.tableUniqueName) { return false; - } else if (!tableUniqueName.equals(other.tableUniqueName)) { + } + + if (!tableUniqueName.equals(other.tableUniqueName)) { return false; } return true; http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java index e92563b..57e1544 100644 --- a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java @@ -129,10 +129,6 @@ public class CarbonUpdateUtil { AbsoluteTableIdentifier absoluteTableIdentifier = table.getAbsoluteTableIdentifier(); - CarbonTablePath carbonTablePath = CarbonStorePath - .getCarbonTablePath(absoluteTableIdentifier.getStorePath(), - absoluteTableIdentifier.getCarbonTableIdentifier()); - // read the existing file if present and update the same. SegmentUpdateDetails[] oldDetails = segmentUpdateStatusManager .getUpdateStatusDetails(); http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedRawResultCollector.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedRawResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedRawResultCollector.java index 479a684..ea89ce5 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedRawResultCollector.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedRawResultCollector.java @@ -200,8 +200,7 @@ public class RestructureBasedRawResultCollector extends RawBasedResultCollector if (null != defaultValue) { defaultValueAsLong = ((Integer) defaultValue).longValue(); } else { - defaultValueAsLong = - new Integer(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY).longValue(); + defaultValueAsLong = (long)CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY; } keyArrayWithNewAddedColumns[newKeyArrayIndex++] = defaultValueAsLong; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ComplexQueryType.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ComplexQueryType.java b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ComplexQueryType.java index 080d577..2274186 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ComplexQueryType.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ComplexQueryType.java @@ -20,12 +20,9 @@ package org.apache.carbondata.core.scan.complextypes; import java.io.IOException; import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk; -import org.apache.carbondata.core.scan.filter.GenericQueryType; import org.apache.carbondata.core.scan.processor.BlocksChunkHolder; public class ComplexQueryType { - protected GenericQueryType children; - protected String name; protected String parentname; http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java index f03f6ea..ee04bb6 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java @@ -775,8 +775,7 @@ public class QueryUtil { new StructQueryType(dimension.getColName(), dimension.getColName(), dimensionToBlockIndexMap.get(dimension.getOrdinal())); complexTypeMap.put(dimension.getOrdinal(), parentQueryType); - parentBlockIndex = - fillChildrenDetails(eachComplexColumnValueSize, columnIdToDictionaryMap, parentBlockIndex, + fillChildrenDetails(eachComplexColumnValueSize, columnIdToDictionaryMap, parentBlockIndex, dimension, parentQueryType); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java index e035970..aed2775 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java @@ -175,10 +175,10 @@ public class RestructureUtil { // 2. If default value is not specified then its surrogate key will be // 1 which is for member default value null if (isDefaultValueNull(defaultValue)) { - dictionaryDefaultValue = new Integer(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY); + dictionaryDefaultValue = CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY; } else { dictionaryDefaultValue = - new Integer(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY + 1); + CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY + 1; } return dictionaryDefaultValue; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/scan/expression/RangeExpressionEvaluator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/RangeExpressionEvaluator.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/RangeExpressionEvaluator.java index 4433e86..d98593c 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/expression/RangeExpressionEvaluator.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/RangeExpressionEvaluator.java @@ -19,6 +19,7 @@ package org.apache.carbondata.core.scan.expression; import java.util.ArrayList; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; @@ -109,9 +110,12 @@ public class RangeExpressionEvaluator { Map<String, List<FilterModificationNode>> filterExpressionMap) { List<FilterModificationNode> deleteExp = new ArrayList<>(); - for (String colName : filterExpressionMap.keySet()) { - // Check is there are multiple list for this Column. - List<FilterModificationNode> filterExp = filterExpressionMap.get(colName); + Iterator<Map.Entry<String, List<FilterModificationNode>>> iterator = + filterExpressionMap.entrySet().iterator(); + Map.Entry<String, List<FilterModificationNode>> nextEntry = null; + while (iterator.hasNext()) { + nextEntry = iterator.next(); + List<FilterModificationNode> filterExp = nextEntry.getValue(); if (filterExp.size() > 1) { // There are multiple Expression for the same column traverse and check if they can // form a range. @@ -120,8 +124,8 @@ public class RangeExpressionEvaluator { for (FilterModificationNode exp : filterExp) { if ((exp.getExpType() == GREATERTHAN) || (exp.getExpType() == GREATERTHAN_EQUALTO)) { - if ((null == endMax) || checkLiteralValue(exp.getCurrentExp(), - endMax.getCurrentExp())) { + if ((null == endMax) || ((checkLiteralValue(exp.getCurrentExp(), + endMax.getCurrentExp())))) { if (null == startMin) { startMin = exp; } else { @@ -141,8 +145,8 @@ public class RangeExpressionEvaluator { } } if ((exp.getExpType() == LESSTHAN) || (exp.getExpType() == LESSTHAN_EQUALTO)) { - if ((null == startMin) || checkLiteralValue(exp.getCurrentExp(), - startMin.getCurrentExp())) { + if ((null == startMin) || ((checkLiteralValue(exp.getCurrentExp(), + startMin.getCurrentExp())))) { if (null == endMax) { endMax = exp; } else { @@ -166,7 +170,7 @@ public class RangeExpressionEvaluator { if ((null != startMin) && (null != endMax)) { LOG.info( "GreaterThan and LessThan Filter Expression changed to Range Expression for column " - + colName); + + nextEntry.getKey()); // the node can be converted to RANGE. Expression n1 = startMin.getCurrentExp(); Expression n2 = endMax.getCurrentExp(); http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/ListExpression.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/ListExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/ListExpression.java index 32ca11c..1c00b14 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/ListExpression.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/ListExpression.java @@ -48,13 +48,18 @@ public class ListExpression extends Expression { } @Override public ExpressionType getFilterExpressionType() { - // TODO Auto-generated method stub return ExpressionType.LIST; } @Override public String getString() { - // TODO Auto-generated method stub - return null; + StringBuffer value = new StringBuffer(); + value.append("ListExpression("); + for (Expression expr : children) { + value.append(expr.getString()).append(";"); + } + value.append(')'); + + return value.toString(); } @Override public void findAndSetChild(Expression oldExpr, Expression newExpr) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java index 3a19e1b..9bc864d 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java @@ -163,18 +163,17 @@ public class FilterExpressionProcessor implements FilterProcessor { } private String joinByteArray(byte[] bytes, char delimiter) { - String byteArrayAsString = ""; + StringBuffer byteArrayAsString = new StringBuffer(); + byteArrayAsString.append(""); if (null != bytes) { for (int i = 0; i < bytes.length; i++) { - byteArrayAsString = byteArrayAsString + delimiter + bytes[i]; + byteArrayAsString.append(delimiter).append(bytes[i]); } if (byteArrayAsString.length() > 0) { - byteArrayAsString = byteArrayAsString.substring(1); + return byteArrayAsString.substring(1); } - } else { - byteArrayAsString = null; } - return byteArrayAsString; + return null; } /** @@ -539,9 +538,9 @@ public class FilterExpressionProcessor implements FilterProcessor { if ((condExpression.getColumnList().get(0).getCarbonColumn() .hasEncoding(Encoding.DICTIONARY) && !condExpression.getColumnList().get(0) .getCarbonColumn().hasEncoding(Encoding.DIRECT_DICTIONARY)) - || (currentCondExpression.getColumnList().get(0).getCarbonColumn().isMeasure())) { + || (condExpression.getColumnList().get(0).getCarbonColumn().isMeasure())) { return new ConditionalFilterResolverImpl(expression, true, true, tableIdentifier, - currentCondExpression.getColumnList().get(0).getCarbonColumn().isMeasure()); + condExpression.getColumnList().get(0).getCarbonColumn().isMeasure()); } } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java index 2ead304..6230771 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java @@ -205,7 +205,7 @@ public final class FilterUtil { segmentProperties, true); } else { return new RestructureIncludeFilterExecutorImpl(dimColResolvedFilterInfo, - msrColResolvedFilterInfo, segmentProperties, true); + msrColResolvedFilterInfo, true); } } if (null != dimColResolvedFilterInfo && dimColResolvedFilterInfo.getDimension().isColumnar()) { @@ -221,7 +221,7 @@ public final class FilterUtil { segmentProperties, false); } else { return new RestructureIncludeFilterExecutorImpl(dimColResolvedFilterInfo, - msrColResolvedFilterInfo, segmentProperties, false); + msrColResolvedFilterInfo, false); } } else { return new IncludeColGroupFilterExecuterImpl(dimColResolvedFilterInfo, segmentProperties); @@ -254,7 +254,7 @@ public final class FilterUtil { segmentProperties, true); } else { return new RestructureExcludeFilterExecutorImpl(dimColResolvedFilterInfo, - msrColResolvedFilterInfo, segmentProperties, true); + msrColResolvedFilterInfo, true); } } if ((null != dimColResolvedFilterInfo) && (dimColResolvedFilterInfo.getDimension() @@ -271,7 +271,7 @@ public final class FilterUtil { segmentProperties, false); } else { return new RestructureExcludeFilterExecutorImpl(dimColResolvedFilterInfo, - msrColResolvedFilterInfo, segmentProperties, false); + msrColResolvedFilterInfo, false); } } else { return new ExcludeColGroupFilterExecuterImpl(dimColResolvedFilterInfo, segmentProperties); http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java index 8e06894..1509949 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java @@ -19,7 +19,6 @@ package org.apache.carbondata.core.scan.filter.executer; import java.io.IOException; import java.util.BitSet; -import org.apache.carbondata.core.datastore.block.SegmentProperties; import org.apache.carbondata.core.scan.filter.FilterUtil; import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo; import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo; @@ -28,10 +27,6 @@ import org.apache.carbondata.core.util.BitSetGroup; public class RestructureExcludeFilterExecutorImpl extends RestructureEvaluatorImpl { - protected DimColumnResolvedFilterInfo dimColEvaluatorInfo; - protected MeasureColumnResolvedFilterInfo measureColumnResolvedFilterInfo; - protected SegmentProperties segmentProperties; - protected boolean isMeasure; /** * flag to check whether filter values contain the default value applied on the dimension column @@ -40,12 +35,7 @@ public class RestructureExcludeFilterExecutorImpl extends RestructureEvaluatorIm protected boolean isDefaultValuePresentInFilterValues; public RestructureExcludeFilterExecutorImpl(DimColumnResolvedFilterInfo dimColEvaluatorInfo, - MeasureColumnResolvedFilterInfo measureColumnResolvedFilterInfo, - SegmentProperties segmentProperties, boolean isMeasure) { - this.dimColEvaluatorInfo = dimColEvaluatorInfo; - this.measureColumnResolvedFilterInfo = measureColumnResolvedFilterInfo; - this.segmentProperties = segmentProperties; - this.isMeasure = isMeasure; + MeasureColumnResolvedFilterInfo measureColumnResolvedFilterInfo, boolean isMeasure) { if (isMeasure) { isDefaultValuePresentInFilterValues = isMeasureDefaultValuePresentInFilterValues(measureColumnResolvedFilterInfo); http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java index 20889ea..0e9a5f0 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java @@ -19,7 +19,6 @@ package org.apache.carbondata.core.scan.filter.executer; import java.io.IOException; import java.util.BitSet; -import org.apache.carbondata.core.datastore.block.SegmentProperties; import org.apache.carbondata.core.scan.filter.FilterUtil; import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo; import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo; @@ -28,11 +27,6 @@ import org.apache.carbondata.core.util.BitSetGroup; public class RestructureIncludeFilterExecutorImpl extends RestructureEvaluatorImpl { - protected DimColumnResolvedFilterInfo dimColumnEvaluatorInfo; - protected MeasureColumnResolvedFilterInfo measureColumnResolvedFilterInfo; - protected SegmentProperties segmentProperties; - protected boolean isMeasure; - /** * flag to check whether filter values contain the default value applied on the dimension column * which does not exist in the current block @@ -40,12 +34,7 @@ public class RestructureIncludeFilterExecutorImpl extends RestructureEvaluatorIm protected boolean isDefaultValuePresentInFilterValues; public RestructureIncludeFilterExecutorImpl(DimColumnResolvedFilterInfo dimColumnEvaluatorInfo, - MeasureColumnResolvedFilterInfo measureColumnResolvedFilterInfo, - SegmentProperties segmentProperties, boolean isMeasure) { - this.dimColumnEvaluatorInfo = dimColumnEvaluatorInfo; - this.measureColumnResolvedFilterInfo = measureColumnResolvedFilterInfo; - this.isMeasure = isMeasure; - this.segmentProperties = segmentProperties; + MeasureColumnResolvedFilterInfo measureColumnResolvedFilterInfo, boolean isMeasure) { if (isMeasure) { isDefaultValuePresentInFilterValues = isMeasureDefaultValuePresentInFilterValues(measureColumnResolvedFilterInfo); http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java index 40065f1..e066ee3 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java @@ -110,7 +110,7 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter { } else { this.dimColEvaluatorInfoList = dimColEvaluatorInfoList; } - if (dimColEvaluatorInfoList.size() > 0) { + if (this.dimColEvaluatorInfoList.size() > 0) { this.isDimensionPresentInCurrentBlock = new boolean[dimColEvaluatorInfoList.size()]; this.dimensionBlocksIndex = new int[dimColEvaluatorInfoList.size()]; } else { @@ -122,7 +122,7 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter { } else { this.msrColEvalutorInfoList = msrColEvalutorInfoList; } - if (msrColEvalutorInfoList.size() > 0) { + if (this.msrColEvalutorInfoList.size() > 0) { this.isMeasurePresentInCurrentBlock = new boolean[msrColEvalutorInfoList.size()]; this.measureBlocksIndex = new int[msrColEvalutorInfoList.size()]; } else { http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerImpl.java index cb9495b..32b1ae1 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerImpl.java @@ -21,11 +21,9 @@ import org.apache.carbondata.core.scan.expression.Expression; public class FilterOptimizerImpl implements FilterOptimizer { - protected FilterOptimizer filterOptimizer; protected Expression filterExpression; - public FilterOptimizerImpl(FilterOptimizer filterOptimizer, Expression filterExpression) { - this.filterOptimizer = filterOptimizer; + public FilterOptimizerImpl(Expression filterExpression) { this.filterExpression = filterExpression; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/scan/filter/optimizer/RangeFilterOptmizer.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/optimizer/RangeFilterOptmizer.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/optimizer/RangeFilterOptmizer.java index 6c61b62..e5cb0ad 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/optimizer/RangeFilterOptmizer.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/optimizer/RangeFilterOptmizer.java @@ -27,7 +27,7 @@ public class RangeFilterOptmizer extends FilterOptimizerImpl { RangeExpressionEvaluator rangeExpEvaluator; public RangeFilterOptmizer(FilterOptimizer filterOptimizer, Expression filterExpression) { - super(filterOptimizer, filterExpression); + super(filterExpression); this.rangeExpEvaluator = new RangeExpressionEvaluator(filterExpression); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/MeasureColumnResolvedFilterInfo.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/MeasureColumnResolvedFilterInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/MeasureColumnResolvedFilterInfo.java index 3880eb7..f8586ed 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/MeasureColumnResolvedFilterInfo.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/MeasureColumnResolvedFilterInfo.java @@ -44,8 +44,6 @@ public class MeasureColumnResolvedFilterInfo extends ColumnResolvedFilterInfo private boolean isMeasureExistsInCurrentSilce = true; - private Object defaultValue; - private CarbonColumn carbonColumn; private CarbonMeasure carbonMeasure; @@ -115,10 +113,6 @@ public class MeasureColumnResolvedFilterInfo extends ColumnResolvedFilterInfo return true; } - public Object getDefaultValue() { - return defaultValue; - } - public CarbonColumn getCarbonColumn() { return carbonColumn; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java index b703959..221d243 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java @@ -48,25 +48,28 @@ public class CustomTypeDictionaryVisitor implements ResolvedFilterInfoVisitorInt public void populateFilterResolvedInfo(ColumnResolvedFilterInfo visitableObj, FilterResolverMetadata metadata) throws FilterUnsupportedException { ColumnFilterInfo resolvedFilterObject = null; - DimColumnResolvedFilterInfo resolveDimension = (DimColumnResolvedFilterInfo) visitableObj; - List<String> evaluateResultListFinal; - try { - evaluateResultListFinal = metadata.getExpression().evaluate(null).getListAsString(); - } catch (FilterIllegalMemberException e) { - throw new FilterUnsupportedException(e); + if (visitableObj instanceof DimColumnResolvedFilterInfo) { + DimColumnResolvedFilterInfo resolveDimension = (DimColumnResolvedFilterInfo) visitableObj; + List<String> evaluateResultListFinal; + try { + evaluateResultListFinal = metadata.getExpression().evaluate(null).getListAsString(); + } catch (FilterIllegalMemberException e) { + throw new FilterUnsupportedException(e); + } + resolvedFilterObject = + getDirectDictionaryValKeyMemberForFilter(metadata.getColumnExpression(), + evaluateResultListFinal, metadata.isIncludeFilter(), + metadata.getColumnExpression().getDimension().getDataType()); + if (!metadata.isIncludeFilter() && null != resolvedFilterObject && !resolvedFilterObject + .getFilterList().contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY)) { + // Adding default surrogate key of null member inorder to not display the same while + // displaying the report as per hive compatibility. + resolvedFilterObject.getFilterList() + .add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY); + Collections.sort(resolvedFilterObject.getFilterList()); + } + resolveDimension.setFilterValues(resolvedFilterObject); } - resolvedFilterObject = getDirectDictionaryValKeyMemberForFilter(metadata.getColumnExpression(), - evaluateResultListFinal, metadata.isIncludeFilter(), - metadata.getColumnExpression().getDimension().getDataType()); - if (!metadata.isIncludeFilter() && null != resolvedFilterObject && !resolvedFilterObject - .getFilterList().contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY)) { - // Adding default surrogate key of null member inorder to not display the same while - // displaying the report as per hive compatibility. - resolvedFilterObject.getFilterList() - .add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY); - Collections.sort(resolvedFilterObject.getFilterList()); - } - resolveDimension.setFilterValues(resolvedFilterObject); } protected ColumnFilterInfo getDirectDictionaryValKeyMemberForFilter( http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java index b13b8d3..2fc7503 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java @@ -45,30 +45,32 @@ public class DictionaryColumnVisitor implements ResolvedFilterInfoVisitorIntf { public void populateFilterResolvedInfo(ColumnResolvedFilterInfo visitableObj, FilterResolverMetadata metadata) throws FilterUnsupportedException, IOException { - DimColumnResolvedFilterInfo resolveDimension = (DimColumnResolvedFilterInfo) visitableObj; - ColumnFilterInfo resolvedFilterObject = null; - List<String> evaluateResultListFinal; - try { - evaluateResultListFinal = metadata.getExpression().evaluate(null).getListAsString(); - } catch (FilterIllegalMemberException e) { - throw new FilterUnsupportedException(e); - } - resolvedFilterObject = FilterUtil - .getFilterValues(metadata.getTableIdentifier(), metadata.getColumnExpression(), - evaluateResultListFinal, metadata.isIncludeFilter()); - if (!metadata.isIncludeFilter() && null != resolvedFilterObject) { - // Adding default surrogate key of null member inorder to not display the same while - // displaying the report as per hive compatibility. - // first check of surrogate key for null value is already added then - // no need to add again otherwise result will be wrong in case of exclude filter - // this is because two times it will flip the same bit - if (!resolvedFilterObject.getFilterList() - .contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY)) { - resolvedFilterObject.getFilterList() - .add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY); + if (visitableObj instanceof DimColumnResolvedFilterInfo) { + DimColumnResolvedFilterInfo resolveDimension = (DimColumnResolvedFilterInfo) visitableObj; + ColumnFilterInfo resolvedFilterObject = null; + List<String> evaluateResultListFinal; + try { + evaluateResultListFinal = metadata.getExpression().evaluate(null).getListAsString(); + } catch (FilterIllegalMemberException e) { + throw new FilterUnsupportedException(e); + } + resolvedFilterObject = FilterUtil + .getFilterValues(metadata.getTableIdentifier(), metadata.getColumnExpression(), + evaluateResultListFinal, metadata.isIncludeFilter()); + if (!metadata.isIncludeFilter() && null != resolvedFilterObject) { + // Adding default surrogate key of null member inorder to not display the same while + // displaying the report as per hive compatibility. + // first check of surrogate key for null value is already added then + // no need to add again otherwise result will be wrong in case of exclude filter + // this is because two times it will flip the same bit + if (!resolvedFilterObject.getFilterList() + .contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY)) { + resolvedFilterObject.getFilterList() + .add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY); + } + Collections.sort(resolvedFilterObject.getFilterList()); } - Collections.sort(resolvedFilterObject.getFilterList()); + resolveDimension.setFilterValues(resolvedFilterObject); } - resolveDimension.setFilterValues(resolvedFilterObject); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/MeasureColumnVisitor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/MeasureColumnVisitor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/MeasureColumnVisitor.java index 8c86a2b..ac4ced8 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/MeasureColumnVisitor.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/MeasureColumnVisitor.java @@ -45,33 +45,35 @@ public class MeasureColumnVisitor implements ResolvedFilterInfoVisitorIntf { */ public void populateFilterResolvedInfo(ColumnResolvedFilterInfo visitableObj, FilterResolverMetadata metadata) throws FilterUnsupportedException { - MeasureColumnResolvedFilterInfo resolveDimension = - (MeasureColumnResolvedFilterInfo) visitableObj; - ColumnFilterInfo resolvedFilterObject = null; - List<String> evaluateResultListFinal = null; - try { - // handling for is null case scenarios - if (metadata.getExpression() instanceof EqualToExpression) { - EqualToExpression expression = (EqualToExpression) metadata.getExpression(); - if (expression.isNull) { - evaluateResultListFinal = new ArrayList<>(1); + if (visitableObj instanceof MeasureColumnResolvedFilterInfo) { + MeasureColumnResolvedFilterInfo resolveDimension = + (MeasureColumnResolvedFilterInfo) visitableObj; + ColumnFilterInfo resolvedFilterObject = null; + List<String> evaluateResultListFinal = new ArrayList<>(1); + try { + // handling for is null case scenarios + if (metadata.getExpression() instanceof EqualToExpression) { + EqualToExpression expression = (EqualToExpression) metadata.getExpression(); + if (expression.isNull) { + evaluateResultListFinal = new ArrayList<>(1); + evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL); + } + } else { + evaluateResultListFinal = metadata.getExpression().evaluate(null).getListAsString(); + } + // Adding default null member inorder to not display the same while + // displaying the report as per hive compatibility. + if (!metadata.isIncludeFilter() && !evaluateResultListFinal + .contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) { evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL); } - } else { - evaluateResultListFinal = metadata.getExpression().evaluate(null).getListAsString(); - } - // Adding default null member inorder to not display the same while - // displaying the report as per hive compatibility. - if (!metadata.isIncludeFilter() && !evaluateResultListFinal - .contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) { - evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL); + } catch (FilterIllegalMemberException e) { + throw new FilterUnsupportedException(e); } - } catch (FilterIllegalMemberException e) { - throw new FilterUnsupportedException(e); + resolvedFilterObject = FilterUtil + .getMeasureValKeyMemberForFilter(evaluateResultListFinal, metadata.isIncludeFilter(), + metadata.getColumnExpression().getDataType(), resolveDimension.getMeasure()); + resolveDimension.setFilterValues(resolvedFilterObject); } - resolvedFilterObject = FilterUtil - .getMeasureValKeyMemberForFilter(evaluateResultListFinal, metadata.isIncludeFilter(), - metadata.getColumnExpression().getDataType(), resolveDimension.getMeasure()); - resolveDimension.setFilterValues(resolvedFilterObject); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java index a8af72e..132f854 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java @@ -45,32 +45,34 @@ public class NoDictionaryTypeVisitor implements ResolvedFilterInfoVisitorIntf { */ public void populateFilterResolvedInfo(ColumnResolvedFilterInfo visitableObj, FilterResolverMetadata metadata) throws FilterUnsupportedException { - DimColumnResolvedFilterInfo resolveDimension = (DimColumnResolvedFilterInfo) visitableObj; - ColumnFilterInfo resolvedFilterObject = null; - List<String> evaluateResultListFinal = null; - try { - // handling for is null case scenarios - if (metadata.getExpression() instanceof EqualToExpression) { - EqualToExpression expression = (EqualToExpression) metadata.getExpression(); - if (expression.isNull) { - evaluateResultListFinal = new ArrayList<>(1); + if (visitableObj instanceof DimColumnResolvedFilterInfo) { + DimColumnResolvedFilterInfo resolveDimension = (DimColumnResolvedFilterInfo) visitableObj; + ColumnFilterInfo resolvedFilterObject = null; + List<String> evaluateResultListFinal = new ArrayList<>(1); + try { + // handling for is null case scenarios + if (metadata.getExpression() instanceof EqualToExpression) { + EqualToExpression expression = (EqualToExpression) metadata.getExpression(); + if (expression.isNull) { + evaluateResultListFinal = new ArrayList<>(1); + evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL); + } + } else { + evaluateResultListFinal = metadata.getExpression().evaluate(null).getListAsString(); + } + // Adding default null member inorder to not display the same while + // displaying the report as per hive compatibility. + if (!metadata.isIncludeFilter() && !evaluateResultListFinal + .contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) { evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL); } - } else { - evaluateResultListFinal = metadata.getExpression().evaluate(null).getListAsString(); - } - // Adding default null member inorder to not display the same while - // displaying the report as per hive compatibility. - if (!metadata.isIncludeFilter() && !evaluateResultListFinal - .contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) { - evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL); + } catch (FilterIllegalMemberException e) { + throw new FilterUnsupportedException(e); } - } catch (FilterIllegalMemberException e) { - throw new FilterUnsupportedException(e); + resolvedFilterObject = FilterUtil + .getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, metadata.isIncludeFilter(), + metadata.getColumnExpression().getDataType()); + resolveDimension.setFilterValues(resolvedFilterObject); } - resolvedFilterObject = FilterUtil - .getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, metadata.isIncludeFilter(), - metadata.getColumnExpression().getDataType()); - resolveDimension.setFilterValues(resolvedFilterObject); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDictionaryColumnVisitor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDictionaryColumnVisitor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDictionaryColumnVisitor.java index 41c95e5..3d16cd2 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDictionaryColumnVisitor.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDictionaryColumnVisitor.java @@ -44,20 +44,22 @@ public class RangeDictionaryColumnVisitor extends DictionaryColumnVisitor */ public void populateFilterResolvedInfo(ColumnResolvedFilterInfo visitableObj, FilterResolverMetadata metadata) throws FilterUnsupportedException, IOException { - DimColumnResolvedFilterInfo resolveDimension = (DimColumnResolvedFilterInfo) visitableObj; - ColumnFilterInfo resolvedFilterObject = null; - List<String> evaluateResultListFinal; - resolvedFilterObject = FilterUtil - .getFilterListForAllValues(metadata.getTableIdentifier(), metadata.getExpression(), - metadata.getColumnExpression(), metadata.isIncludeFilter()); + if (visitableObj instanceof DimColumnResolvedFilterInfo) { + DimColumnResolvedFilterInfo resolveDimension = (DimColumnResolvedFilterInfo) visitableObj; + ColumnFilterInfo resolvedFilterObject = null; + List<String> evaluateResultListFinal; + resolvedFilterObject = FilterUtil + .getFilterListForAllValues(metadata.getTableIdentifier(), metadata.getExpression(), + metadata.getColumnExpression(), metadata.isIncludeFilter()); - if (!metadata.isIncludeFilter() && null != resolvedFilterObject) { - // Adding default surrogate key of null member inorder to not display the same while - // displaying the report as per hive compatibility. - resolvedFilterObject.getFilterList() - .add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY); - Collections.sort(resolvedFilterObject.getFilterList()); + if (!metadata.isIncludeFilter() && null != resolvedFilterObject) { + // Adding default surrogate key of null member inorder to not display the same while + // displaying the report as per hive compatibility. + resolvedFilterObject.getFilterList() + .add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY); + Collections.sort(resolvedFilterObject.getFilterList()); + } + resolveDimension.setFilterValues(resolvedFilterObject); } - resolveDimension.setFilterValues(resolvedFilterObject); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDirectDictionaryVisitor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDirectDictionaryVisitor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDirectDictionaryVisitor.java index d740648..918fd9e 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDirectDictionaryVisitor.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDirectDictionaryVisitor.java @@ -46,36 +46,39 @@ public class RangeDirectDictionaryVisitor extends CustomTypeDictionaryVisitor */ public void populateFilterResolvedInfo(ColumnResolvedFilterInfo visitableObj, FilterResolverMetadata metadata) throws FilterUnsupportedException { - DimColumnResolvedFilterInfo resolveDimension = (DimColumnResolvedFilterInfo) visitableObj; - ColumnFilterInfo resolvedFilterObject = null; - List<ExpressionResult> listOfExpressionResults = new ArrayList<ExpressionResult>(20); - List<String> evaluateResultListFinal = new ArrayList<String>(); - try { - listOfExpressionResults = ((RangeExpression) metadata.getExpression()).getLiterals(); + if (visitableObj instanceof DimColumnResolvedFilterInfo) { + DimColumnResolvedFilterInfo resolveDimension = (DimColumnResolvedFilterInfo) visitableObj; + ColumnFilterInfo resolvedFilterObject = null; + List<ExpressionResult> listOfExpressionResults = null; + List<String> evaluateResultListFinal = new ArrayList<String>(); + try { + listOfExpressionResults = ((RangeExpression) metadata.getExpression()).getLiterals(); - for (ExpressionResult result : listOfExpressionResults) { - if (result.getString() == null) { - evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL); - continue; + for (ExpressionResult result : listOfExpressionResults) { + if (result.getString() == null) { + evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL); + continue; + } + evaluateResultListFinal.add(result.getString()); } - evaluateResultListFinal.add(result.getString()); + } catch (FilterIllegalMemberException e) { + throw new FilterUnsupportedException(e); } - } catch (FilterIllegalMemberException e) { - throw new FilterUnsupportedException(e); - } - resolvedFilterObject = getDirectDictionaryValKeyMemberForFilter(metadata.getColumnExpression(), - evaluateResultListFinal, metadata.isIncludeFilter(), - metadata.getColumnExpression().getDimension().getDataType()); + resolvedFilterObject = + getDirectDictionaryValKeyMemberForFilter(metadata.getColumnExpression(), + evaluateResultListFinal, metadata.isIncludeFilter(), + metadata.getColumnExpression().getDimension().getDataType()); - if (!metadata.isIncludeFilter() && null != resolvedFilterObject && !resolvedFilterObject - .getFilterList().contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY)) { - // Adding default surrogate key of null member inorder to not display the same while - // displaying the report as per hive compatibility. - resolvedFilterObject.getFilterList() - .add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY); - Collections.sort(resolvedFilterObject.getFilterList()); + if (!metadata.isIncludeFilter() && null != resolvedFilterObject && !resolvedFilterObject + .getFilterList().contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY)) { + // Adding default surrogate key of null member inorder to not display the same while + // displaying the report as per hive compatibility. + resolvedFilterObject.getFilterList() + .add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY); + Collections.sort(resolvedFilterObject.getFilterList()); + } + resolveDimension.setFilterValues(resolvedFilterObject); } - resolveDimension.setFilterValues(resolvedFilterObject); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeNoDictionaryTypeVisitor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeNoDictionaryTypeVisitor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeNoDictionaryTypeVisitor.java index 87864f7..d4c44ba 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeNoDictionaryTypeVisitor.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeNoDictionaryTypeVisitor.java @@ -47,35 +47,37 @@ public class RangeNoDictionaryTypeVisitor extends NoDictionaryTypeVisitor */ public void populateFilterResolvedInfo(ColumnResolvedFilterInfo visitableObj, FilterResolverMetadata metadata) throws FilterUnsupportedException { - DimColumnResolvedFilterInfo resolveDimension = (DimColumnResolvedFilterInfo) visitableObj; - ColumnFilterInfo resolvedFilterObject = null; - List<ExpressionResult> listOfExpressionResults = new ArrayList<ExpressionResult>(20); - List<String> evaluateResultListFinal = new ArrayList<String>(); - try { - // Add The Range Filter Values. - if (metadata.getExpression() instanceof RangeExpression) { - listOfExpressionResults = ((RangeExpression) metadata.getExpression()).getLiterals(); - } + if (visitableObj instanceof DimColumnResolvedFilterInfo) { + DimColumnResolvedFilterInfo resolveDimension = (DimColumnResolvedFilterInfo) visitableObj; + ColumnFilterInfo resolvedFilterObject = null; + List<ExpressionResult> listOfExpressionResults = new ArrayList<ExpressionResult>(20); + List<String> evaluateResultListFinal = new ArrayList<String>(); + try { + // Add The Range Filter Values. + if (metadata.getExpression() instanceof RangeExpression) { + listOfExpressionResults = ((RangeExpression) metadata.getExpression()).getLiterals(); + } - for (ExpressionResult result : listOfExpressionResults) { - if (result.getString() == null) { - evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL); - continue; + for (ExpressionResult result : listOfExpressionResults) { + if (result.getString() == null) { + evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL); + continue; + } + evaluateResultListFinal.add(result.getString()); } - evaluateResultListFinal.add(result.getString()); - } - // evaluateResultListFinal.add(metadata.getExpression().evaluate().getListAsString()); - if (!metadata.isIncludeFilter() && !evaluateResultListFinal - .contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) { - evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL); + // evaluateResultListFinal.add(metadata.getExpression().evaluate().getListAsString()); + if (!metadata.isIncludeFilter() && !evaluateResultListFinal + .contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) { + evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL); + } + } catch (FilterIllegalMemberException e) { + throw new FilterUnsupportedException(e); } - } catch (FilterIllegalMemberException e) { - throw new FilterUnsupportedException(e); + resolvedFilterObject = FilterUtil + .getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, metadata.isIncludeFilter(), + metadata.getColumnExpression().getDataType()); + resolveDimension.setFilterValues(resolvedFilterObject); } - resolvedFilterObject = FilterUtil - .getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, metadata.isIncludeFilter(), - metadata.getColumnExpression().getDataType()); - resolveDimension.setFilterValues(resolvedFilterObject); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/scan/result/vector/ColumnVectorInfo.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/ColumnVectorInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/ColumnVectorInfo.java index 631ab97..a5f81b9 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/ColumnVectorInfo.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/ColumnVectorInfo.java @@ -36,4 +36,27 @@ public class ColumnVectorInfo implements Comparable<ColumnVectorInfo> { @Override public int compareTo(ColumnVectorInfo o) { return ordinal - o.ordinal; } + + @Override public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (null == obj) { + return false; + } + + if (!(obj instanceof ColumnVectorInfo)) { + return false; + } + + return ordinal == ((ColumnVectorInfo) obj).ordinal; + } + + @Override public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + (ordinal); + return result; + } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsRecorderImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsRecorderImpl.java b/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsRecorderImpl.java index ffb7d7f..9807eaa 100644 --- a/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsRecorderImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsRecorderImpl.java @@ -164,14 +164,16 @@ public class QueryStatisticsRecorderImpl implements QueryStatisticsRecorder, Ser values.add(String.valueOf(result_size)); StringBuilder tableInfo = new StringBuilder(); String[] columns = headers.split(","); - String line = ""; - String hearLine = ""; - String valueLine = ""; + StringBuilder line = new StringBuilder(""); + StringBuilder hearLine = new StringBuilder(""); + StringBuilder valueLine = new StringBuilder(""); for (int i = 0; i < columns.length; i++) { int len = Math.max(columns[i].length(), values.get(i).length()); - line += "+" + printLine("-", len); - hearLine += "|" + printLine(" ", len - columns[i].length()) + columns[i]; - valueLine += "|" + printLine(" ", len - values.get(i).length()) + values.get(i); + line.append("+").append(printLine("-", len)); + hearLine.append("|").append(printLine(" ", len - columns[i].length())) + .append(columns[i]); + valueLine.append("|").append(printLine(" ", len - values.get(i).length())) + .append(values.get(i)); } // struct table info tableInfo.append(line).append("+").append("\n"); http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java index 831a2fb..d1c16bb 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java @@ -143,8 +143,8 @@ public final class ByteUtil { * 32-bit. On the other hand, it is substantially faster on 64-bit. */ for (int i = 0; i < minWords * SIZEOF_LONG; i += SIZEOF_LONG) { - long lw = CarbonUnsafe.unsafe.getLong(buffer1, offset1Adj + (long) i); - long rw = CarbonUnsafe.unsafe.getLong(buffer2, offset2Adj + (long) i); + long lw = CarbonUnsafe.getUnsafe().getLong(buffer1, offset1Adj + (long) i); + long rw = CarbonUnsafe.getUnsafe().getLong(buffer2, offset2Adj + (long) i); long diff = lw ^ rw; if (diff != 0) { @@ -209,8 +209,10 @@ public final class ByteUtil { if (minLength > 7) { minWords = minLength / SIZEOF_LONG; for (int i = 0; i < minWords * SIZEOF_LONG; i += SIZEOF_LONG) { - long lw = CarbonUnsafe.unsafe.getLong(buffer1, CarbonUnsafe.BYTE_ARRAY_OFFSET + (long) i); - long rw = CarbonUnsafe.unsafe.getLong(buffer2, CarbonUnsafe.BYTE_ARRAY_OFFSET + (long) i); + long lw = + CarbonUnsafe.getUnsafe().getLong(buffer1, CarbonUnsafe.BYTE_ARRAY_OFFSET + (long) i); + long rw = + CarbonUnsafe.getUnsafe().getLong(buffer2, CarbonUnsafe.BYTE_ARRAY_OFFSET + (long) i); long diff = lw ^ rw; if (diff != 0) { @@ -252,6 +254,7 @@ public final class ByteUtil { } return len1 - len2; } + public boolean equals(byte[] buffer1, byte[] buffer2) { if (buffer1.length != buffer2.length) { return false; @@ -259,8 +262,8 @@ public final class ByteUtil { int len = buffer1.length / 8; long currentOffset = CarbonUnsafe.BYTE_ARRAY_OFFSET; for (int i = 0; i < len; i++) { - long lw = CarbonUnsafe.unsafe.getLong(buffer1, currentOffset); - long rw = CarbonUnsafe.unsafe.getLong(buffer2, currentOffset); + long lw = CarbonUnsafe.getUnsafe().getLong(buffer1, currentOffset); + long rw = CarbonUnsafe.getUnsafe().getLong(buffer2, currentOffset); if (lw != rw) { return false; } @@ -269,8 +272,8 @@ public final class ByteUtil { len = buffer1.length % 8; if (len > 0) { for (int i = 0; i < len; i += 1) { - long lw = CarbonUnsafe.unsafe.getByte(buffer1, currentOffset); - long rw = CarbonUnsafe.unsafe.getByte(buffer2, currentOffset); + long lw = CarbonUnsafe.getUnsafe().getByte(buffer1, currentOffset); + long rw = CarbonUnsafe.getUnsafe().getByte(buffer2, currentOffset); if (lw != rw) { return false; } @@ -288,8 +291,8 @@ public final class ByteUtil { int len = length1 / 8; long currentOffset = CarbonUnsafe.BYTE_ARRAY_OFFSET; for (int i = 0; i < len; i++) { - long lw = CarbonUnsafe.unsafe.getLong(buffer1, currentOffset + offset1); - long rw = CarbonUnsafe.unsafe.getLong(buffer2, currentOffset + offset2); + long lw = CarbonUnsafe.getUnsafe().getLong(buffer1, currentOffset + offset1); + long rw = CarbonUnsafe.getUnsafe().getLong(buffer2, currentOffset + offset2); if (lw != rw) { return false; } @@ -298,8 +301,8 @@ public final class ByteUtil { len = buffer1.length % 8; if (len > 0) { for (int i = 0; i < len; i += 1) { - long lw = CarbonUnsafe.unsafe.getByte(buffer1, currentOffset + offset1); - long rw = CarbonUnsafe.unsafe.getByte(buffer2, currentOffset + offset2); + long lw = CarbonUnsafe.getUnsafe().getByte(buffer1, currentOffset + offset1); + long rw = CarbonUnsafe.getUnsafe().getByte(buffer2, currentOffset + offset2); if (lw != rw) { return false; } @@ -426,12 +429,12 @@ public final class ByteUtil { throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_SHORT); } short n = 0; - if (CarbonUnsafe.unsafe != null) { + if (CarbonUnsafe.getUnsafe() != null) { if (CarbonUnsafe.ISLITTLEENDIAN) { n = Short.reverseBytes( - CarbonUnsafe.unsafe.getShort(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET)); + CarbonUnsafe.getUnsafe().getShort(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET)); } else { - n = CarbonUnsafe.unsafe.getShort(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET); + n = CarbonUnsafe.getUnsafe().getShort(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET); } } else { @@ -497,12 +500,12 @@ public final class ByteUtil { throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_INT); } int n = 0; - if (CarbonUnsafe.unsafe != null) { + if (CarbonUnsafe.getUnsafe() != null) { if (CarbonUnsafe.ISLITTLEENDIAN) { n = Integer.reverseBytes( - CarbonUnsafe.unsafe.getInt(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET)); + CarbonUnsafe.getUnsafe().getInt(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET)); } else { - n = CarbonUnsafe.unsafe.getInt(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET); + n = CarbonUnsafe.getUnsafe().getInt(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET); } } else { for (int i = offset; i < (offset + length); i++) { @@ -550,12 +553,12 @@ public final class ByteUtil { throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_LONG); } long l = 0; - if (CarbonUnsafe.unsafe != null) { + if (CarbonUnsafe.getUnsafe() != null) { if (CarbonUnsafe.ISLITTLEENDIAN) { l = Long.reverseBytes( - CarbonUnsafe.unsafe.getLong(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET)); + CarbonUnsafe.getUnsafe().getLong(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET)); } else { - l = CarbonUnsafe.unsafe.getLong(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET); + l = CarbonUnsafe.getUnsafe().getLong(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET); } } else { for (int i = offset; i < offset + length; i++) { @@ -594,11 +597,11 @@ public final class ByteUtil { "Not enough room to put an int at" + " offset " + offset + " in a " + bytes.length + " byte array"); } - if (CarbonUnsafe.unsafe != null) { + if (CarbonUnsafe.getUnsafe() != null) { if (CarbonUnsafe.ISLITTLEENDIAN) { val = Integer.reverseBytes(val); } - CarbonUnsafe.unsafe.putInt(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET, val); + CarbonUnsafe.getUnsafe().putInt(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET, val); return offset + ByteUtil.SIZEOF_INT; } else { for (int i = offset + 3; i > offset; i--) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java index 669d3f2..061721b 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java +++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java @@ -137,22 +137,18 @@ public final class CarbonProperties { } private void validateLockType() { - String lockTypeConfigured = carbonProperties.getProperty(CarbonCommonConstants.LOCK_TYPE); - if (null != lockTypeConfigured) { - lockTypeConfigured = lockTypeConfigured.toUpperCase(); - switch (lockTypeConfigured) { - // if user is setting the lock type as CARBON_LOCK_TYPE_ZOOKEEPER then no need to validate - // else validate based on the file system type for LOCAL file system lock will be - // CARBON_LOCK_TYPE_LOCAL and for the distributed one CARBON_LOCK_TYPE_HDFS - case CarbonCommonConstants.CARBON_LOCK_TYPE_ZOOKEEPER: - break; - case CarbonCommonConstants.CARBON_LOCK_TYPE_LOCAL: - case CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS: - default: - validateAndConfigureLockType(lockTypeConfigured); - } - } else { - validateAndConfigureLockType(lockTypeConfigured); + String lockTypeConfigured = carbonProperties + .getProperty(CarbonCommonConstants.LOCK_TYPE, CarbonCommonConstants.LOCK_TYPE_DEFAULT); + switch (lockTypeConfigured.toUpperCase()) { + // if user is setting the lock type as CARBON_LOCK_TYPE_ZOOKEEPER then no need to validate + // else validate based on the file system type for LOCAL file system lock will be + // CARBON_LOCK_TYPE_LOCAL and for the distributed one CARBON_LOCK_TYPE_HDFS + case CarbonCommonConstants.CARBON_LOCK_TYPE_ZOOKEEPER: + break; + case CarbonCommonConstants.CARBON_LOCK_TYPE_LOCAL: + case CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS: + default: + validateAndConfigureLockType(lockTypeConfigured); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java index 15512a8..3bf837b 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java @@ -304,8 +304,9 @@ public final class CarbonUtil { */ private static void deleteRecursive(File f) throws IOException { if (f.isDirectory()) { - if (f.listFiles() != null) { - for (File c : f.listFiles()) { + File[] files = f.listFiles(); + if (null != files) { + for (File c : files) { deleteRecursive(c); } } @@ -950,6 +951,7 @@ public final class CarbonUtil { fileReader.finish(); } catch (IOException e) { // ignore the exception as nothing we can do about it + fileReader = null; } } } @@ -1438,7 +1440,8 @@ public final class CarbonUtil { return valueEncoderMeta; } - public static byte[] serializeEncodeMetaUsingByteBuffer(ValueEncoderMeta valueEncoderMeta) { + public static byte[] serializeEncodeMetaUsingByteBuffer(ValueEncoderMeta valueEncoderMeta) + throws IOException { ByteBuffer buffer = null; switch (valueEncoderMeta.getType()) { case LONG: @@ -1463,6 +1466,8 @@ public final class CarbonUtil { buffer = ByteBuffer.allocate(CarbonCommonConstants.INT_SIZE_IN_BYTE + 3); buffer.putChar(valueEncoderMeta.getTypeInChar()); break; + default: + throw new IOException("Unsupported datatype: " + valueEncoderMeta.getType()); } buffer.putInt(0); // decimal point, not used buffer.put(valueEncoderMeta.getDataTypeSelected()); http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/util/DataTypeConverterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeConverterImpl.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeConverterImpl.java index c71a4d0..afc81bf 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeConverterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeConverterImpl.java @@ -35,7 +35,7 @@ public class DataTypeConverterImpl implements DataTypeConverter, Serializable { } public byte[] convertFromStringToByte(Object data) { - return data.toString().getBytes(); + return data.toString().getBytes(CarbonCommonConstants.DEFAULT_CHARSET_CLASS); } public Object convertFromStringToUTF8String(Object data) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java index 6e54f1f..2cd7ce5 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java @@ -154,7 +154,11 @@ public final class DataTypeUtil { bigDecimalMsrValue = bigDecimalMsrValue.setScale(carbonMeasure.getScale(), RoundingMode.HALF_UP); } - return normalizeDecimalValue(bigDecimalMsrValue, carbonMeasure.getPrecision()); + if (null != bigDecimalMsrValue) { + return normalizeDecimalValue(bigDecimalMsrValue, carbonMeasure.getPrecision()); + } else { + return bigDecimalMsrValue; + } default: return measurePage.getDouble(index); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/util/comparator/Comparator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/comparator/Comparator.java b/core/src/main/java/org/apache/carbondata/core/util/comparator/Comparator.java index 1f43c7c..5681364 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/comparator/Comparator.java +++ b/core/src/main/java/org/apache/carbondata/core/util/comparator/Comparator.java @@ -120,13 +120,7 @@ class DoubleSerializableComparator implements SerializableComparator { } else if (key2 == null) { return 1; } - if ((double) key1 < (double) key2) { - return -1; - } else if ((double) key1 > (double) key2) { - return 1; - } else { - return 0; - } + return ((Double)key1).compareTo((Double)key2); } }
