http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java b/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java index d3b9b48..2115f82 100644 --- a/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java +++ b/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java @@ -159,7 +159,8 @@ public class UnsafeMemoryManager { /** * It tries to allocate memory of `size` bytes, keep retry until it allocates successfully. */ - public static MemoryBlock allocateMemoryWithRetry(long taskId, long size) throws MemoryException { + public static MemoryBlock allocateMemoryWithRetry(long taskId, long size) + throws MemoryException { MemoryBlock baseBlock = null; int tries = 0; while (tries < 300) { @@ -177,8 +178,7 @@ public class UnsafeMemoryManager { tries++; } if (baseBlock == null) { - LOGGER.error(" Memory Used : " + INSTANCE.memoryUsed + " Tasks running : " - + taskIdToMemoryBlockMap.keySet()); + INSTANCE.printCurrentMemoryUsage(); throw new MemoryException("Not enough memory"); } return baseBlock; @@ -187,4 +187,9 @@ public class UnsafeMemoryManager { public static boolean isOffHeap() { return offHeap; } + + private synchronized void printCurrentMemoryUsage() { + LOGGER.error( + " Memory Used : " + memoryUsed + " Tasks running : " + taskIdToMemoryBlockMap.keySet()); + } }
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ArrayType.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ArrayType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ArrayType.java index 0f0f120..94a4e89 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ArrayType.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ArrayType.java @@ -31,20 +31,32 @@ public class ArrayType extends DataType { return true; } - public DataType getElementType() { - return elementType; + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (!(obj instanceof ArrayType)) { + return false; + } + if (!this.getName().equalsIgnoreCase(((ArrayType) obj).getName())) { + return false; + } + return true; } - @Override public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof ArrayType)) return false; - - ArrayType arrayType = (ArrayType) o; - - return elementType.equals(arrayType.elementType); + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + getName().hashCode(); + return result; } - @Override public int hashCode() { - return elementType.hashCode(); + public DataType getElementType() { + return elementType; } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalType.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalType.java index b2acd21..8536222 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalType.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalType.java @@ -29,6 +29,37 @@ public class DecimalType extends DataType { this.scale = scale; } + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (!(obj instanceof DecimalType)) { + return false; + } + if (!this.getName().equalsIgnoreCase(((DecimalType) obj).getName())) { + return false; + } + if (this.precision != ((DecimalType) obj).precision) { + return false; + } + if (this.scale != ((DecimalType) obj).scale) { + return false; + } + return true; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + getName().hashCode(); + return result; + } + public int getPrecision() { return precision; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StructType.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StructType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StructType.java index 97cc4f0..90b7374 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StructType.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StructType.java @@ -33,6 +33,31 @@ public class StructType extends DataType { return true; } + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (!(obj instanceof StructType)) { + return false; + } + if (!this.getName().equalsIgnoreCase(((StructType) obj).getName())) { + return false; + } + return true; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + getName().hashCode(); + return result; + } + public List<StructField> getFields() { return fields; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java index 80c6a3a..fb4d8e3 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java @@ -33,15 +33,11 @@ import org.apache.carbondata.core.metadata.schema.table.Writable; import org.apache.carbondata.core.metadata.schema.table.WritableUtil; import org.apache.carbondata.core.preagg.TimeSeriesUDF; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - /** * Store the information about the column meta data present the table */ public class ColumnSchema implements Serializable, Writable { - private static final Log LOG = LogFactory.getLog(ColumnSchema.class); /** * serialization version */ @@ -334,8 +330,6 @@ public class ColumnSchema implements Serializable, Writable { return false; } } else if (!columnName.equals(other.columnName)) { - LOG.error("column name is " + columnName - + " but other column name is " + other.columnName); return false; } if (dataType == null) { @@ -343,8 +337,6 @@ public class ColumnSchema implements Serializable, Writable { return false; } } else if (!dataType.equals(other.dataType)) { - LOG.error("column name is" + columnName + " data type is " + dataType - + " but other column data type is " + other.dataType); return false; } return true; @@ -361,40 +353,16 @@ public class ColumnSchema implements Serializable, Writable { return false; } ColumnSchema other = (ColumnSchema) obj; - if (!columnUniqueId.equals(other.columnUniqueId)) { - LOG.error("Index file's column " + columnName + " columnUniqueId is " + columnUniqueId - + " but table's column columnUniqueId is " + other.columnUniqueId); - return false; - } - if (isDimensionColumn != other.isDimensionColumn) { - LOG.error("Index file's column " + columnName + " isDimensionColumn is " + isDimensionColumn - + " but table's column isDimensionColumn is " + other.isDimensionColumn); - return false; - } - if (scale != other.scale) { - LOG.error("Index file's column " + columnName + " scale is " + scale - + " but table's column scale is " + other.scale); - return false; - } - if (precision != other.precision) { - LOG.error("Index file's column " + columnName + " precision is " + precision - + " but table's column precision is " + other.precision); - return false; - } - if (isSortColumn != other.isSortColumn) { - LOG.error("Index file's column " + columnName + " isSortColumn is " + isSortColumn - + " but table's column isSortColumn is " + other.isSortColumn); + if (!columnUniqueId.equals(other.columnUniqueId) || + (isDimensionColumn != other.isDimensionColumn) || + (isSortColumn != other.isSortColumn)) { return false; } if (encodingList.size() != other.encodingList.size()) { - LOG.error("Index file's column " + columnName + " encoding size is " + encodingList.size() - + " but table's column encoding size is " + other.encodingList.size()); return false; } for (int i = 0; i < encodingList.size(); i++) { if (encodingList.get(i).compareTo(other.encodingList.get(i)) != 0) { - LOG.error("Index file's column " + columnName + " encoding is " + encodingList.get(i) - + " but table's column encoding is " + other.encodingList.get(i)); return false; } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/preagg/TimeSeriesUDF.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/preagg/TimeSeriesUDF.java b/core/src/main/java/org/apache/carbondata/core/preagg/TimeSeriesUDF.java index df712de..ce0f15d 100644 --- a/core/src/main/java/org/apache/carbondata/core/preagg/TimeSeriesUDF.java +++ b/core/src/main/java/org/apache/carbondata/core/preagg/TimeSeriesUDF.java @@ -112,7 +112,7 @@ public class TimeSeriesUDF { /** * Below method will be used to initialize the thread local */ - private synchronized void initialize() { + private void initialize() { if (calanderThreadLocal.get() == null) { calanderThreadLocal.set(new GregorianCalendar()); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java index 6439b36..4f41b92 100644 --- a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java @@ -111,6 +111,9 @@ public class CarbonDictionaryMetadataReaderImpl implements CarbonDictionaryMetad // get the thrift object for dictionary chunk dictionaryChunkMeta = (ColumnDictionaryChunkMeta) dictionaryMetadataFileReader.read(); } + if (null == dictionaryChunkMeta) { + throw new IOException("Last dictionary chunk does not exist"); + } // create a new instance of chunk meta wrapper using thrift object return getNewInstanceOfCarbonDictionaryColumnMetaChunk(dictionaryChunkMeta); } @@ -128,6 +131,9 @@ public class CarbonDictionaryMetadataReaderImpl implements CarbonDictionaryMetad break; } } + if (null == dictionaryChunkMeta) { + throw new IOException("Matching dictionary chunk does not exist"); + } // create a new instance of chunk meta wrapper using thrift object return getNewInstanceOfCarbonDictionaryColumnMetaChunk(dictionaryChunkMeta); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedRawResultCollector.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedRawResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedRawResultCollector.java index 1c440cf..33f0db7 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedRawResultCollector.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedRawResultCollector.java @@ -197,7 +197,7 @@ public class RestructureBasedRawResultCollector extends RawBasedResultCollector if (CarbonUtil.hasEncoding(actualQueryDimensions[i].getDimension().getEncoder(), Encoding.DICTIONARY)) { // if dimension exists then add the key array value else add the default value - if (dimensionInfo.getDimensionExists()[i]) { + if (dimensionInfo.getDimensionExists()[i] && null != keyArray && 0 != keyArray.length) { keyArrayWithNewAddedColumns[newKeyArrayIndex++] = keyArray[existingColumnKeyArrayIndex++]; } else { http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/SearchModeDetailQueryExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/SearchModeDetailQueryExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/SearchModeDetailQueryExecutor.java index 04669ab..aed472c 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/SearchModeDetailQueryExecutor.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/SearchModeDetailQueryExecutor.java @@ -70,9 +70,7 @@ public class SearchModeDetailQueryExecutor extends AbstractQueryExecutor<Object> public CarbonIterator<Object> execute(QueryModel queryModel) throws QueryExecutionException, IOException { List<BlockExecutionInfo> blockExecutionInfoList = getBlockExecutionInfos(queryModel); - if (executorService == null) { - initThreadPool(); - } + this.queryIterator = new SearchModeResultIterator( blockExecutionInfoList, queryModel, http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/SearchModeVectorDetailQueryExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/SearchModeVectorDetailQueryExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/SearchModeVectorDetailQueryExecutor.java index 6c9396b..00fd511 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/SearchModeVectorDetailQueryExecutor.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/SearchModeVectorDetailQueryExecutor.java @@ -74,9 +74,7 @@ public class SearchModeVectorDetailQueryExecutor extends AbstractQueryExecutor<O public CarbonIterator<Object> execute(QueryModel queryModel) throws QueryExecutionException, IOException { List<BlockExecutionInfo> blockExecutionInfoList = getBlockExecutionInfos(queryModel); - if (executorService == null) { - initThreadPool(); - } + this.queryIterator = new SearchModeVectorResultIterator( blockExecutionInfoList, queryModel, http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java index dc9415e..74c9ae2 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java @@ -207,12 +207,14 @@ public class FilterExpressionProcessor implements FilterProcessor { */ private void addBlockBasedOnMinMaxValue(FilterExecuter filterExecuter, List<DataRefNode> listOfDataBlocksToScan, DataRefNode dataRefNode) { - + if (null == dataRefNode.getColumnsMinValue() || null == dataRefNode.getColumnsMaxValue()) { + listOfDataBlocksToScan.add(dataRefNode); + return; + } BitSet bitSet = filterExecuter .isScanRequired(dataRefNode.getColumnsMaxValue(), dataRefNode.getColumnsMinValue()); if (!bitSet.isEmpty()) { listOfDataBlocksToScan.add(dataRefNode); - } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java index 5196f8f..9741915 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java @@ -77,12 +77,10 @@ import org.apache.carbondata.core.scan.expression.logical.AndExpression; import org.apache.carbondata.core.scan.expression.logical.TrueExpression; import org.apache.carbondata.core.scan.filter.executer.AndFilterExecuterImpl; import org.apache.carbondata.core.scan.filter.executer.DimColumnExecuterFilterInfo; -import org.apache.carbondata.core.scan.filter.executer.ExcludeColGroupFilterExecuterImpl; import org.apache.carbondata.core.scan.filter.executer.ExcludeFilterExecuterImpl; import org.apache.carbondata.core.scan.filter.executer.FalseFilterExecutor; import org.apache.carbondata.core.scan.filter.executer.FilterExecuter; import org.apache.carbondata.core.scan.filter.executer.ImplicitIncludeFilterExecutorImpl; -import org.apache.carbondata.core.scan.filter.executer.IncludeColGroupFilterExecuterImpl; import org.apache.carbondata.core.scan.filter.executer.IncludeFilterExecuterImpl; import org.apache.carbondata.core.scan.filter.executer.MeasureColumnExecuterFilterInfo; import org.apache.carbondata.core.scan.filter.executer.OrFilterExecuterImpl; @@ -232,30 +230,24 @@ public final class FilterUtil { msrColResolvedFilterInfo, true); } } - if (null != dimColResolvedFilterInfo) { - CarbonDimension dimension = dimColResolvedFilterInfo.getDimension(); - if (dimension.hasEncoding(Encoding.IMPLICIT)) { - return new ImplicitIncludeFilterExecutorImpl(dimColResolvedFilterInfo); - } else if (dimension.isColumnar()) { - CarbonDimension dimensionFromCurrentBlock = - segmentProperties.getDimensionFromCurrentBlock(dimColResolvedFilterInfo.getDimension()); - if (null != dimensionFromCurrentBlock) { - // update dimension and column index according to the dimension position in current block - DimColumnResolvedFilterInfo dimColResolvedFilterInfoCopyObject = - dimColResolvedFilterInfo.getCopyObject(); - dimColResolvedFilterInfoCopyObject.setDimension(dimensionFromCurrentBlock); - dimColResolvedFilterInfoCopyObject.setColumnIndex(dimensionFromCurrentBlock.getOrdinal()); - return new IncludeFilterExecuterImpl(dimColResolvedFilterInfoCopyObject, null, - segmentProperties, false); - } else { - return new RestructureIncludeFilterExecutorImpl(dimColResolvedFilterInfo, - msrColResolvedFilterInfo, false); - } + CarbonDimension dimension = dimColResolvedFilterInfo.getDimension(); + if (dimension.hasEncoding(Encoding.IMPLICIT)) { + return new ImplicitIncludeFilterExecutorImpl(dimColResolvedFilterInfo); + } else { + CarbonDimension dimensionFromCurrentBlock = + segmentProperties.getDimensionFromCurrentBlock(dimColResolvedFilterInfo.getDimension()); + if (null != dimensionFromCurrentBlock) { + // update dimension and column index according to the dimension position in current block + DimColumnResolvedFilterInfo dimColResolvedFilterInfoCopyObject = + dimColResolvedFilterInfo.getCopyObject(); + dimColResolvedFilterInfoCopyObject.setDimension(dimensionFromCurrentBlock); + dimColResolvedFilterInfoCopyObject.setColumnIndex(dimensionFromCurrentBlock.getOrdinal()); + return new IncludeFilterExecuterImpl(dimColResolvedFilterInfoCopyObject, null, + segmentProperties, false); } else { - return new IncludeColGroupFilterExecuterImpl(dimColResolvedFilterInfo, segmentProperties); + return new RestructureIncludeFilterExecutorImpl(dimColResolvedFilterInfo, + msrColResolvedFilterInfo, false); } - } else { - return new IncludeColGroupFilterExecuterImpl(null, segmentProperties); } } @@ -288,24 +280,19 @@ public final class FilterUtil { msrColResolvedFilterInfo, true); } } - if ((null != dimColResolvedFilterInfo) && (dimColResolvedFilterInfo.getDimension() - .isColumnar())) { - CarbonDimension dimensionFromCurrentBlock = - segmentProperties.getDimensionFromCurrentBlock(dimColResolvedFilterInfo.getDimension()); - if (null != dimensionFromCurrentBlock) { - // update dimension and column index according to the dimension position in current block - DimColumnResolvedFilterInfo dimColResolvedFilterInfoCopyObject = - dimColResolvedFilterInfo.getCopyObject(); - dimColResolvedFilterInfoCopyObject.setDimension(dimensionFromCurrentBlock); - dimColResolvedFilterInfoCopyObject.setColumnIndex(dimensionFromCurrentBlock.getOrdinal()); - return new ExcludeFilterExecuterImpl(dimColResolvedFilterInfoCopyObject, null, - segmentProperties, false); - } else { - return new RestructureExcludeFilterExecutorImpl(dimColResolvedFilterInfo, - msrColResolvedFilterInfo, false); - } + CarbonDimension dimensionFromCurrentBlock = + segmentProperties.getDimensionFromCurrentBlock(dimColResolvedFilterInfo.getDimension()); + if (null != dimensionFromCurrentBlock) { + // update dimension and column index according to the dimension position in current block + DimColumnResolvedFilterInfo dimColResolvedFilterInfoCopyObject = + dimColResolvedFilterInfo.getCopyObject(); + dimColResolvedFilterInfoCopyObject.setDimension(dimensionFromCurrentBlock); + dimColResolvedFilterInfoCopyObject.setColumnIndex(dimensionFromCurrentBlock.getOrdinal()); + return new ExcludeFilterExecuterImpl(dimColResolvedFilterInfoCopyObject, null, + segmentProperties, false); } else { - return new ExcludeColGroupFilterExecuterImpl(dimColResolvedFilterInfo, segmentProperties); + return new RestructureExcludeFilterExecutorImpl(dimColResolvedFilterInfo, + msrColResolvedFilterInfo, false); } } @@ -581,17 +568,15 @@ public final class FilterUtil { } Collections.sort(surrogates); ColumnFilterInfo columnFilterInfo = null; - if (surrogates.size() > 0) { - columnFilterInfo = new ColumnFilterInfo(); - if (isExcludeFilterNeedsToApply) { - columnFilterInfo.setOptimized(true); - } - columnFilterInfo.setIncludeFilter(isIncludeFilter); - if (!isIncludeFilter) { - columnFilterInfo.setExcludeFilterList(surrogates); - } else { - columnFilterInfo.setFilterList(surrogates); - } + columnFilterInfo = new ColumnFilterInfo(); + if (isExcludeFilterNeedsToApply) { + columnFilterInfo.setOptimized(true); + } + columnFilterInfo.setIncludeFilter(isIncludeFilter); + if (!isIncludeFilter) { + columnFilterInfo.setExcludeFilterList(surrogates); + } else { + columnFilterInfo.setFilterList(surrogates); } return columnFilterInfo; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java deleted file mode 100644 index 44f7c07..0000000 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.carbondata.core.scan.filter.executer; - -import java.util.BitSet; - -import org.apache.carbondata.core.datastore.block.SegmentProperties; -import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo; - -/** - * It checks if filter is required on given block and if required, it does - * linear search on block data and set the bitset. - */ -public class ExcludeColGroupFilterExecuterImpl extends ExcludeFilterExecuterImpl { - - /** - * @param dimColResolvedFilterInfo - * @param segmentProperties - */ - public ExcludeColGroupFilterExecuterImpl(DimColumnResolvedFilterInfo dimColResolvedFilterInfo, - SegmentProperties segmentProperties) { - super(dimColResolvedFilterInfo, null, segmentProperties, false); - } - - /** - * Check if scan is required on given block based on min and max value - */ - public BitSet isScanRequired(byte[][] blkMaxVal, byte[][] blkMinVal) { - BitSet bitSet = new BitSet(1); - bitSet.flip(0, 1); - return bitSet; - } - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java deleted file mode 100644 index e4da26f..0000000 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java +++ /dev/null @@ -1,232 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.carbondata.core.scan.filter.executer; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.BitSet; -import java.util.List; - -import org.apache.carbondata.common.logging.LogService; -import org.apache.carbondata.common.logging.LogServiceFactory; -import org.apache.carbondata.core.datastore.block.SegmentProperties; -import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage; -import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk; -import org.apache.carbondata.core.keygenerator.KeyGenException; -import org.apache.carbondata.core.keygenerator.KeyGenerator; -import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo; -import org.apache.carbondata.core.scan.executor.util.QueryUtil; -import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo; -import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks; -import org.apache.carbondata.core.util.BitSetGroup; -import org.apache.carbondata.core.util.ByteUtil; - -/** - * It checks if filter is required on given block and if required, it does - * linear search on block data and set the bitset. - */ -public class IncludeColGroupFilterExecuterImpl extends IncludeFilterExecuterImpl { - - /** - * LOGGER - */ - private static final LogService LOGGER = - LogServiceFactory.getLogService(IncludeColGroupFilterExecuterImpl.class.getName()); - - /** - * @param dimColResolvedFilterInfo - * @param segmentProperties - */ - public IncludeColGroupFilterExecuterImpl(DimColumnResolvedFilterInfo dimColResolvedFilterInfo, - SegmentProperties segmentProperties) { - super(dimColResolvedFilterInfo, null, segmentProperties, false); - } - - /** - * It fills BitSet with row index which matches filter key - */ - protected BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage, - int numerOfRows) { - BitSet bitSet = new BitSet(numerOfRows); - - try { - KeyStructureInfo keyStructureInfo = getKeyStructureInfo(); - byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys(); - for (int i = 0; i < filterValues.length; i++) { - byte[] filterVal = filterValues[i]; - for (int rowId = 0; rowId < numerOfRows; rowId++) { - byte[] colData = new byte[keyStructureInfo.getMaskByteRanges().length]; - dimensionColumnPage.fillRawData(rowId, 0, colData, keyStructureInfo); - if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterVal, colData) == 0) { - bitSet.set(rowId); - } - } - } - - } catch (Exception e) { - LOGGER.error(e); - } - - return bitSet; - } - - @Override - public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks, - boolean useBitsetPipeLine) throws IOException { - int chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping() - .get(dimColumnEvaluatorInfo.getColumnIndex()); - if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) { - rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] = - rawBlockletColumnChunks.getDataBlock().readDimensionChunk( - rawBlockletColumnChunks.getFileReader(), chunkIndex); - } - DimensionRawColumnChunk dimensionRawColumnChunk = - rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]; - BitSetGroup bitSetGroup = new BitSetGroup(dimensionRawColumnChunk.getPagesCount()); - for (int i = 0; i < dimensionRawColumnChunk.getPagesCount(); i++) { - if (dimensionRawColumnChunk.getMaxValues() != null) { - BitSet bitSet = getFilteredIndexes(dimensionRawColumnChunk.decodeColumnPage(i), - dimensionRawColumnChunk.getRowCount()[i]); - bitSetGroup.setBitSet(bitSet, i); - } - } - return bitSetGroup; - } - - /** - * It is required for extracting column data from columngroup chunk - * - * @return - * @throws KeyGenException - */ - private KeyStructureInfo getKeyStructureInfo() throws KeyGenException { - int colGrpId = getColumnGroupId(dimColumnEvaluatorInfo.getColumnIndex()); - KeyGenerator keyGenerator = segmentProperties.getColumnGroupAndItsKeygenartor().get(colGrpId); - List<Integer> mdKeyOrdinal = new ArrayList<Integer>(); - mdKeyOrdinal.add(getMdkeyOrdinal(dimColumnEvaluatorInfo.getColumnIndex(), colGrpId)); - int[] maskByteRanges = QueryUtil.getMaskedByteRangeBasedOrdinal(mdKeyOrdinal, keyGenerator); - byte[] maxKey = QueryUtil.getMaxKeyBasedOnOrinal(mdKeyOrdinal, keyGenerator); - KeyStructureInfo restructureInfos = new KeyStructureInfo(); - restructureInfos.setKeyGenerator(keyGenerator); - restructureInfos.setMaskByteRanges(maskByteRanges); - restructureInfos.setMaxKey(maxKey); - return restructureInfos; - } - - /** - * Check if scan is required on given block based on min and max value - */ - public BitSet isScanRequired(byte[][] blkMaxVal, byte[][] blkMinVal) { - BitSet bitSet = new BitSet(1); - byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys(); - int columnIndex = dimColumnEvaluatorInfo.getColumnIndex(); - int chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping().get(columnIndex); - int[] cols = getAllColumns(columnIndex); - byte[] maxValue = getMinMaxData(cols, blkMaxVal[chunkIndex], columnIndex); - byte[] minValue = getMinMaxData(cols, blkMinVal[chunkIndex], columnIndex); - boolean isScanRequired = false; - for (int k = 0; k < filterValues.length; k++) { - // filter value should be in range of max and min value i.e - // max>filtervalue>min - // so filter-max should be negative - int maxCompare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], maxValue); - // and filter-min should be positive - int minCompare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], minValue); - - // if any filter value is in range than this block needs to be - // scanned - if (maxCompare <= 0 && minCompare >= 0) { - isScanRequired = true; - break; - } - } - if (isScanRequired) { - bitSet.set(0); - } - return bitSet; - } - - /** - * It extract min and max data for given column from stored min max value - * - * @param colGrpColumns - * @param minMaxData - * @param columnIndex - * @return - */ - private byte[] getMinMaxData(int[] colGrpColumns, byte[] minMaxData, int columnIndex) { - int startIndex = 0; - int endIndex = 0; - if (null != colGrpColumns) { - for (int i = 0; i < colGrpColumns.length; i++) { - int colGrpId = getColumnGroupId(colGrpColumns[i]); - int mdKeyOrdinal = getMdkeyOrdinal(colGrpColumns[i], colGrpId); - int[] byteRange = getKeyGenerator(colGrpId).getKeyByteOffsets(mdKeyOrdinal); - int colSize = 0; - for (int j = byteRange[0]; j <= byteRange[1]; j++) { - colSize++; - } - if (colGrpColumns[i] == columnIndex) { - endIndex = startIndex + colSize; - break; - } - startIndex += colSize; - } - } - byte[] data = new byte[endIndex - startIndex]; - System.arraycopy(minMaxData, startIndex, data, 0, data.length); - return data; - } - - /** - * It returns column groups which have provided column ordinal - * - * @param columnIndex - * @return column group array - */ - private int[] getAllColumns(int columnIndex) { - int[][] colGroups = segmentProperties.getColumnGroups(); - for (int i = 0; i < colGroups.length; i++) { - if (QueryUtil.searchInArray(colGroups[i], columnIndex)) { - return colGroups[i]; - } - } - return null; - } - - private int getMdkeyOrdinal(int ordinal, int colGrpId) { - return segmentProperties.getColumnGroupMdKeyOrdinal(colGrpId, ordinal); - } - - private int getColumnGroupId(int ordinal) { - int[][] columnGroups = segmentProperties.getColumnGroups(); - int colGrpId = -1; - for (int i = 0; i < columnGroups.length; i++) { - if (columnGroups[i].length > 1) { - colGrpId++; - if (QueryUtil.searchInArray(columnGroups[i], ordinal)) { - break; - } - } - } - return colGrpId; - } - - public KeyGenerator getKeyGenerator(int colGrpId) { - return segmentProperties.getColumnGroupAndItsKeygenartor().get(colGrpId); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java index 34555e1..b218813 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java @@ -113,7 +113,7 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl { */ private void ifDefaultValueMatchesFilter() { isDefaultValuePresentInFilter = false; - if (!this.isDimensionPresentInCurrentBlock) { + if (!this.isDimensionPresentInCurrentBlock && null != filterRangesValues) { CarbonDimension dimension = this.dimColEvaluatorInfo.getDimension(); byte[] defaultValue = dimension.getDefaultValue(); if (null != defaultValue) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureEvaluatorImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureEvaluatorImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureEvaluatorImpl.java index e1432b0..f901238 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureEvaluatorImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureEvaluatorImpl.java @@ -134,5 +134,4 @@ public abstract class RestructureEvaluatorImpl implements FilterExecuter { } return isDefaultValuePresentInFilterValues; } - } http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java index 4bee89b..057a244 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java @@ -124,8 +124,8 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter { this.dimColEvaluatorInfoList = dimColEvaluatorInfoList; } if (this.dimColEvaluatorInfoList.size() > 0) { - this.isDimensionPresentInCurrentBlock = new boolean[dimColEvaluatorInfoList.size()]; - this.dimensionChunkIndex = new int[dimColEvaluatorInfoList.size()]; + this.isDimensionPresentInCurrentBlock = new boolean[this.dimColEvaluatorInfoList.size()]; + this.dimensionChunkIndex = new int[this.dimColEvaluatorInfoList.size()]; } else { this.isDimensionPresentInCurrentBlock = new boolean[]{false}; this.dimensionChunkIndex = new int[]{0}; @@ -136,8 +136,8 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter { this.msrColEvalutorInfoList = msrColEvalutorInfoList; } if (this.msrColEvalutorInfoList.size() > 0) { - this.isMeasurePresentInCurrentBlock = new boolean[msrColEvalutorInfoList.size()]; - this.measureChunkIndex = new int[msrColEvalutorInfoList.size()]; + this.isMeasurePresentInCurrentBlock = new boolean[this.msrColEvalutorInfoList.size()]; + this.measureChunkIndex = new int[this.msrColEvalutorInfoList.size()]; } else { this.isMeasurePresentInCurrentBlock = new boolean[]{false}; this.measureChunkIndex = new int[] {0}; @@ -647,13 +647,11 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter { } } - if (null != msrColEvalutorInfoList) { - for (MeasureColumnResolvedFilterInfo msrColumnEvalutorInfo : msrColEvalutorInfoList) { - if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[measureChunkIndex[0]]) { - rawBlockletColumnChunks.getMeasureRawColumnChunks()[measureChunkIndex[0]] = - rawBlockletColumnChunks.getDataBlock() - .readMeasureChunk(rawBlockletColumnChunks.getFileReader(), measureChunkIndex[0]); - } + for (MeasureColumnResolvedFilterInfo msrColumnEvalutorInfo : msrColEvalutorInfoList) { + if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[measureChunkIndex[0]]) { + rawBlockletColumnChunks.getMeasureRawColumnChunks()[measureChunkIndex[0]] = + rawBlockletColumnChunks.getDataBlock() + .readMeasureChunk(rawBlockletColumnChunks.getFileReader(), measureChunkIndex[0]); } } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java index be871d4..e339a58 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java @@ -67,7 +67,7 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute this.filterRangeValues = filterRangeValues; this.msrFilterRangeValues = msrFilterRangeValues; lastDimensionColOrdinal = segmentProperties.getLastDimensionColOrdinal(); - if (!msrColEvalutorInfoList.isEmpty()) { + if (!this.msrColEvalutorInfoList.isEmpty()) { CarbonMeasure measure = this.msrColEvalutorInfoList.get(0).getMeasure(); comparator = Comparator.getComparatorByDataTypeForMeasure(measure.getDataType()); } @@ -98,9 +98,12 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute } else if (!msrColEvalutorInfoList.isEmpty() && !isMeasurePresentInCurrentBlock[0]) { CarbonMeasure measure = this.msrColEvalutorInfoList.get(0).getMeasure(); byte[] defaultValue = measure.getDefaultValue(); + SerializableComparator comparatorTmp = (null != comparator ? + comparator : + Comparator.getComparatorByDataTypeForMeasure(measure.getDataType())); if (null != defaultValue) { for (int k = 0; k < msrFilterRangeValues.length; k++) { - int maxCompare = comparator.compare(msrFilterRangeValues[k], + int maxCompare = comparatorTmp.compare(msrFilterRangeValues[k], RestructureUtil.getMeasureDefaultValue(measure.getColumnSchema(), measure.getDefaultValue())); http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java index a3359be..2ea3c73 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java @@ -98,9 +98,11 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte } else if (!msrColEvalutorInfoList.isEmpty() && !isMeasurePresentInCurrentBlock[0]) { CarbonMeasure measure = this.msrColEvalutorInfoList.get(0).getMeasure(); byte[] defaultValue = measure.getDefaultValue(); + SerializableComparator comparatorTmp = + Comparator.getComparatorByDataTypeForMeasure(measure.getDataType()); if (null != defaultValue) { for (int k = 0; k < msrFilterRangeValues.length; k++) { - int maxCompare = comparator.compare(msrFilterRangeValues[k], + int maxCompare = comparatorTmp.compare(msrFilterRangeValues[k], RestructureUtil.getMeasureDefaultValue(measure.getColumnSchema(), measure.getDefaultValue())); if (maxCompare <= 0) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java index 0c268c9..f52d087 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java @@ -100,9 +100,11 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter } else if (!msrColEvalutorInfoList.isEmpty() && !isMeasurePresentInCurrentBlock[0]) { CarbonMeasure measure = this.msrColEvalutorInfoList.get(0).getMeasure(); byte[] defaultValue = measure.getDefaultValue(); + SerializableComparator comparatorTmp = + Comparator.getComparatorByDataTypeForMeasure(measure.getDataType()); if (null != defaultValue) { for (int k = 0; k < msrFilterRangeValues.length; k++) { - int maxCompare = comparator.compare(msrFilterRangeValues[k], + int maxCompare = comparatorTmp.compare(msrFilterRangeValues[k], RestructureUtil.getMeasureDefaultValue(measure.getColumnSchema(), measure.getDefaultValue())); http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFilterExecuterImpl.java index c5ed77d..97e750a 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFilterExecuterImpl.java @@ -100,12 +100,14 @@ public class RowLevelRangeLessThanFilterExecuterImpl extends RowLevelFilterExecu } else if (!msrColEvalutorInfoList.isEmpty() && !isMeasurePresentInCurrentBlock[0]) { CarbonMeasure measure = this.msrColEvalutorInfoList.get(0).getMeasure(); byte[] defaultValue = measure.getDefaultValue(); + SerializableComparator comparatorTmp = + Comparator.getComparatorByDataTypeForMeasure(measure.getDataType()); if (null != defaultValue) { for (int k = 0; k < msrFilterRangeValues.length; k++) { Object convertedValue = RestructureUtil .getMeasureDefaultValue(measure.getColumnSchema(), measure.getDefaultValue()); int maxCompare = - comparator.compare(msrFilterRangeValues[k], convertedValue); + comparatorTmp.compare(msrFilterRangeValues[k], convertedValue); if (maxCompare > 0) { isDefaultValuePresentInFilter = true; break; http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java index ccbbc32..8ad0c48 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java @@ -303,7 +303,6 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf { this.dimColResolvedFilterInfo.getDimension(), segmentProperties, false); } return null; - } http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java index d975c20..01aa939 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java @@ -135,14 +135,19 @@ public abstract class AbstractDetailQueryResultIterator<E> extends CarbonIterato } else { DataRefNode startDataBlock = finder.findFirstDataBlock(dataRefNode, blockInfo.getStartKey()); - while (startDataBlock.nodeIndex() < blockInfo.getStartBlockletIndex()) { + while ((null != startDataBlock) && (startDataBlock.nodeIndex() < blockInfo + .getStartBlockletIndex())) { startDataBlock = startDataBlock.getNextDataRefNode(); } long numberOfBlockToScan = blockInfo.getNumberOfBlockletToScan(); //if number of block is less than 0 then take end block. if (numberOfBlockToScan <= 0) { DataRefNode endDataBlock = finder.findLastDataBlock(dataRefNode, blockInfo.getEndKey()); - numberOfBlockToScan = endDataBlock.nodeIndex() - startDataBlock.nodeIndex() + 1; + if (null != startDataBlock) { + numberOfBlockToScan = endDataBlock.nodeIndex() - startDataBlock.nodeIndex() + 1; + } else { + numberOfBlockToScan = endDataBlock.nodeIndex() + 1; + } } blockInfo.setFirstDataBlock(startDataBlock); blockInfo.setNumberOfBlockToScan(numberOfBlockToScan); http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFilterScanner.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFilterScanner.java b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFilterScanner.java index 57d8177..0100c8b 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFilterScanner.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFilterScanner.java @@ -110,6 +110,10 @@ public class BlockletFilterScanner extends BlockletFullScanner { totalPagesScanned.getCount() + dataBlock.numberOfPages()); // apply min max if (isMinMaxEnabled) { + if (null == dataBlock.getColumnsMaxValue() + || null == dataBlock.getColumnsMinValue()) { + return true; + } BitSet bitSet = null; // check for implicit include filter instance if (filterExecuter instanceof ImplicitColumnFilterExecutor) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java index d6671b4..9dc8fe6 100644 --- a/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java +++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java @@ -230,7 +230,13 @@ public class LoadMetadataDetails implements Serializable { * @return */ public long getLoadStartTimeAsLong() { - return (!loadStartTime.isEmpty()) ? getTimeStamp(loadStartTime) : 0; + if (!loadStartTime.isEmpty()) { + Long time = getTimeStamp(loadStartTime); + if (null != time) { + return time; + } + } + return 0; } /** http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java index 363b5bc..1c53fbb 100644 --- a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java +++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java @@ -322,13 +322,12 @@ public class SegmentUpdateStatusManager { * @return the list of delete file */ private List<String> getDeltaFiles(CarbonFile blockDir, final String blockNameFromTuple, - final String extension, - String segment) { + final String extension, String segment) throws IOException { List<String> deleteFileList = new ArrayList<>(); for (SegmentUpdateDetails block : updateDetails) { - if (block.getBlockName().equalsIgnoreCase(blockNameFromTuple) && - block.getSegmentName().equalsIgnoreCase(segment) && - !CarbonUpdateUtil.isBlockInvalid(block.getSegmentStatus())) { + if (block.getBlockName().equalsIgnoreCase(blockNameFromTuple) && block.getSegmentName() + .equalsIgnoreCase(segment) && !CarbonUpdateUtil + .isBlockInvalid(block.getSegmentStatus())) { final long deltaStartTimestamp = getStartTimeOfDeltaFile(extension, block); // If there is no delete delete file , then return null if (deltaStartTimestamp == 0) { @@ -347,33 +346,38 @@ public class SegmentUpdateStatusManager { private List<String> getFilePaths(CarbonFile blockDir, final String blockNameFromTuple, final String extension, List<String> deleteFileList, final long deltaStartTimestamp, - final long deltaEndTimeStamp) { - CarbonFile[] files = blockDir.getParentFile().listFiles(new CarbonFileFilter() { - - @Override public boolean accept(CarbonFile pathName) { - String fileName = pathName.getName(); - if (fileName.endsWith(extension) && pathName.getSize() > 0) { - String firstPart = fileName.substring(0, fileName.indexOf('.')); - String blockName = - firstPart.substring(0, firstPart.lastIndexOf(CarbonCommonConstants.HYPHEN)); - long timestamp = Long.parseLong(firstPart - .substring(firstPart.lastIndexOf(CarbonCommonConstants.HYPHEN) + 1, - firstPart.length())); - if (blockNameFromTuple.equals(blockName) && ( - (Long.compare(timestamp, deltaEndTimeStamp) <= 0) && ( - Long.compare(timestamp, deltaStartTimestamp) >= 0))) { - return true; + final long deltaEndTimeStamp) throws IOException { + if (null != blockDir.getParentFile()) { + CarbonFile[] files = blockDir.getParentFile().listFiles(new CarbonFileFilter() { + + @Override + public boolean accept(CarbonFile pathName) { + String fileName = pathName.getName(); + if (fileName.endsWith(extension) && pathName.getSize() > 0) { + String firstPart = fileName.substring(0, fileName.indexOf('.')); + String blockName = + firstPart.substring(0, firstPart.lastIndexOf(CarbonCommonConstants.HYPHEN)); + long timestamp = Long.parseLong(firstPart + .substring(firstPart.lastIndexOf(CarbonCommonConstants.HYPHEN) + 1, + firstPart.length())); + if (blockNameFromTuple.equals(blockName) && ( + (Long.compare(timestamp, deltaEndTimeStamp) <= 0) && ( + Long.compare(timestamp, deltaStartTimestamp) >= 0))) { + return true; + } } + return false; } - return false; - } - }); + }); - for (CarbonFile cfile : files) { - if (null == deleteFileList) { - deleteFileList = new ArrayList<String>(files.length); + for (CarbonFile cfile : files) { + if (null == deleteFileList) { + deleteFileList = new ArrayList<String>(files.length); + } + deleteFileList.add(cfile.getCanonicalPath()); } - deleteFileList.add(cfile.getCanonicalPath()); + } else { + throw new IOException("Parent file could not found"); } return deleteFileList; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java index b74c279..e30ad03 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java +++ b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java @@ -60,7 +60,12 @@ public abstract class AbstractDataFileFooterConverter { */ private static BitSet getPresenceMeta( org.apache.carbondata.format.PresenceMeta presentMetadataThrift) { - return BitSet.valueOf(presentMetadataThrift.getPresent_bit_stream()); + final byte[] present_bit_stream = presentMetadataThrift.getPresent_bit_stream(); + if (null != present_bit_stream) { + return BitSet.valueOf(present_bit_stream); + } else { + return new BitSet(1); + } } /** http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java index 9880b4d..af5121c 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java @@ -370,7 +370,7 @@ public class CarbonMetadataUtil { } else if (DataTypes.isDecimal(dataType)) { return DataTypeUtil.byteToBigDecimal(first).compareTo(DataTypeUtil.byteToBigDecimal(second)); } else { - throw new IllegalArgumentException("Invalid data type"); + throw new IllegalArgumentException("Invalid data type:" + dataType); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java index 9822167..f7f71b3 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java @@ -587,20 +587,16 @@ public final class DataTypeUtil { return null; } try { - Object parsedValue = null; if (actualDataType == DataTypes.SHORT) { - parsedValue = Short.parseShort(data); + Short.parseShort(data); } else if (actualDataType == DataTypes.INT) { - parsedValue = Integer.parseInt(data); + Integer.parseInt(data); } else if (actualDataType == DataTypes.LONG) { - parsedValue = Long.parseLong(data); + Long.parseLong(data); } else { return data; } - if (null != parsedValue) { - return data; - } - return null; + return data; } catch (NumberFormatException ex) { return null; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java index 62192ff..e8a121c 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java +++ b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java @@ -576,11 +576,15 @@ public class CarbonTablePath { */ public static CarbonFile[] getSortIndexFiles(CarbonFile sortIndexDir, final String columnUniqueId) { - return sortIndexDir.listFiles(new CarbonFileFilter() { - @Override public boolean accept(CarbonFile file) { - return file.getName().startsWith(columnUniqueId) && file.getName().endsWith(SORT_INDEX_EXT); - } - }); + if (null != sortIndexDir) { + return sortIndexDir.listFiles(new CarbonFileFilter() { + @Override public boolean accept(CarbonFile file) { + return file.getName().startsWith(columnUniqueId) && file.getName() + .endsWith(SORT_INDEX_EXT); + } + }); + } + return null; } /** http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionDataChunkTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionDataChunkTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionDataChunkTest.java deleted file mode 100644 index 54b66a6..0000000 --- a/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionDataChunkTest.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.carbondata.core.datastore.chunk.impl; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import org.apache.carbondata.core.keygenerator.KeyGenException; -import org.apache.carbondata.core.keygenerator.KeyGenerator; -import org.apache.carbondata.core.keygenerator.mdkey.MultiDimKeyVarLengthGenerator; -import org.apache.carbondata.core.util.CarbonUtil; -import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo; -import org.apache.carbondata.core.scan.executor.util.QueryUtil; - -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; - -public class ColumnGroupDimensionDataChunkTest { - - static ColumnGroupDimensionColumnPage columnGroupDimensionDataChunk; - static KeyGenerator keyGenerator; - - @BeforeClass public static void setup() { - int[] bitLength = CarbonUtil.getDimensionBitLength(new int[] { 10, 10, 10 }, new int[] { 3 }); - // create a key generator - keyGenerator = new MultiDimKeyVarLengthGenerator(bitLength); - byte[] data = new byte[keyGenerator.getKeySizeInBytes() * 3]; - int position = 0; - for (int i = 1; i <= 3; i++) { - try { - System.arraycopy(keyGenerator.generateKey(new int[] { i, i, i }), 0, data, position, - keyGenerator.getKeySizeInBytes()); - } catch (KeyGenException e) { - assertTrue(false); - } - position += keyGenerator.getKeySizeInBytes(); - } - columnGroupDimensionDataChunk = - new ColumnGroupDimensionColumnPage(data, keyGenerator.getKeySizeInBytes(), 3); - } - - @Test public void fillChunkDataTest() { - List<Integer> ordinals = new ArrayList<Integer>(); - ordinals.add(1); - KeyStructureInfo keyStructureInfo = getKeyStructureInfo(ordinals, keyGenerator); - byte[] buffer = new byte[1]; - columnGroupDimensionDataChunk.fillRawData(1, 0, buffer, keyStructureInfo); - assertEquals(buffer[0], 2); - } - - @Test public void getChunkDataTest() { - byte[] b = { 34, 2 }; - byte res[] = columnGroupDimensionDataChunk.getChunkData(1); - Assert.assertTrue(Arrays.equals(res, b)); - } - - @Test public void fillConvertedChunkDataTest() { - int[] row = new int[3]; - int[] expected = { 0, 0, 3 }; - List<Integer> ordinals = new ArrayList<Integer>(); - ordinals.add(2); - KeyStructureInfo keyStructureInfo = getKeyStructureInfo(ordinals, keyGenerator); - keyStructureInfo.setMdkeyQueryDimensionOrdinal(new int[] { 2 }); - int res = columnGroupDimensionDataChunk.fillSurrogateKey(2, 2, row, keyStructureInfo); - Assert.assertTrue(Arrays.equals(row, expected)); - } - - /** - * Below method will be used to get the key structure info for the query - * - * @param ordinals query model - * @param keyGenerator - * @return key structure info - */ - private KeyStructureInfo getKeyStructureInfo(List<Integer> ordinals, KeyGenerator keyGenerator) { - // getting the masked byte range for dictionary column - int[] maskByteRanges = QueryUtil.getMaskedByteRangeBasedOrdinal(ordinals, keyGenerator); - - // getting the masked bytes for query dimension dictionary column - int[] maskedBytes = QueryUtil.getMaskedByte(keyGenerator.getKeySizeInBytes(), maskByteRanges); - - // max key for the dictionary dimension present in the query - byte[] maxKey = null; - try { - // getting the max key which will be used to masked and get the - // masked key - maxKey = QueryUtil.getMaxKeyBasedOnOrinal(ordinals, keyGenerator); - } catch (KeyGenException e) { - } - - KeyStructureInfo restructureInfos = new KeyStructureInfo(); - restructureInfos.setKeyGenerator(keyGenerator); - restructureInfos.setMaskByteRanges(maskByteRanges); - restructureInfos.setMaxKey(maxKey); - return restructureInfos; - } -} - http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFileTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFileTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFileTest.java index f55cd67..890d36d 100644 --- a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFileTest.java +++ b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFileTest.java @@ -59,8 +59,11 @@ public class AlluxioCarbonFileTest { } try { FileOutputStream oFile = new FileOutputStream(file, true); + oFile.close(); } catch (FileNotFoundException e) { e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); } fileStatus = new FileStatus(12L, true, 60, 120l, 180L, new Path(file.getAbsolutePath())); http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/ViewFsCarbonFileTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/ViewFsCarbonFileTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/ViewFsCarbonFileTest.java index ba661b1..82d1501 100644 --- a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/ViewFsCarbonFileTest.java +++ b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/ViewFsCarbonFileTest.java @@ -54,14 +54,17 @@ public class ViewFsCarbonFileTest { file = new File("Test.carbondata"); if (!file.exists()) try { - file.createNewFile(); + file.createNewFile(); } catch (IOException e) { - e.printStackTrace(); + e.printStackTrace(); } try { - FileOutputStream oFile = new FileOutputStream(file, true); + FileOutputStream oFile = new FileOutputStream(file, true); + oFile.close(); } catch (FileNotFoundException e) { e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); } fileStatus = new FileStatus(12L, true, 60, 120l, 180L, new Path(file.getAbsolutePath())); http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/hadoop/src/main/java/org/apache/carbondata/hadoop/CacheClient.java ---------------------------------------------------------------------- diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CacheClient.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CacheClient.java index 32af8d3..8be1e2e 100644 --- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CacheClient.java +++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CacheClient.java @@ -16,39 +16,21 @@ */ package org.apache.carbondata.hadoop; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -import org.apache.carbondata.common.logging.LogService; -import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.cache.Cache; import org.apache.carbondata.core.cache.CacheProvider; import org.apache.carbondata.core.cache.CacheType; -import org.apache.carbondata.core.datastore.SegmentTaskIndexStore; import org.apache.carbondata.core.datastore.TableSegmentUniqueIdentifier; -import org.apache.carbondata.core.datastore.block.SegmentProperties; import org.apache.carbondata.core.datastore.block.SegmentTaskIndexWrapper; -import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; -import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema; /** * CacheClient : Holds all the Cache access clients for Btree, Dictionary */ public class CacheClient { - private static final LogService LOGGER = - LogServiceFactory.getLogService(CacheClient.class.getName()); - - private final Object lock = new Object(); - // segment access client for driver LRU cache private CacheAccessClient<TableSegmentUniqueIdentifier, SegmentTaskIndexWrapper> segmentAccessClient; - private static Map<SegmentTaskIndexStore.SegmentPropertiesWrapper, SegmentProperties> - segmentProperties = new ConcurrentHashMap<>(); - public CacheClient() { Cache<TableSegmentUniqueIdentifier, SegmentTaskIndexWrapper> segmentCache = CacheProvider.getInstance().createCache(CacheType.DRIVER_BTREE); @@ -63,35 +45,4 @@ public class CacheClient { public void close() { segmentAccessClient.close(); } - - /** - * Method to get the segment properties and avoid construction of new segment properties until - * the schema is not modified - * - * @param tableIdentifier - * @param columnsInTable - * @param columnCardinality - */ - public SegmentProperties getSegmentProperties(AbsoluteTableIdentifier tableIdentifier, - List<ColumnSchema> columnsInTable, int[] columnCardinality) { - SegmentTaskIndexStore.SegmentPropertiesWrapper segmentPropertiesWrapper = - new SegmentTaskIndexStore.SegmentPropertiesWrapper(tableIdentifier, columnsInTable, - columnCardinality); - SegmentProperties segmentProperties = this.segmentProperties.get(segmentPropertiesWrapper); - if (null == segmentProperties) { - synchronized (lock) { - segmentProperties = this.segmentProperties.get(segmentPropertiesWrapper); - if (null == segmentProperties) { - // create a metadata details - // this will be useful in query handling - // all the data file metadata will have common segment properties we - // can use first one to get create the segment properties - LOGGER.info("Constructing new SegmentProperties"); - segmentProperties = new SegmentProperties(columnsInTable, columnCardinality); - this.segmentProperties.put(segmentPropertiesWrapper, segmentProperties); - } - } - } - return segmentProperties; - } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java ---------------------------------------------------------------------- diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java index 0bcb188..e5e3165 100644 --- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java +++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java @@ -124,6 +124,9 @@ public class CarbonOutputCommitter extends FileOutputCommitter { .mergeSegmentFiles(readPath, segmentFileName, CarbonTablePath.getSegmentFilesLocation(loadModel.getTablePath())); if (segmentFile != null) { + if (null == newMetaEntry) { + throw new RuntimeException("Internal Error"); + } // Move all files from temp directory of each segment to partition directory SegmentFileStore.moveFromTempFolder(segmentFile, loadModel.getSegmentId() + "_" + loadModel.getFactTimeStamp() + ".tmp", http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/Segment.java ---------------------------------------------------------------------- diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/Segment.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/Segment.java index 7d9c712..b619158 100644 --- a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/Segment.java +++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/Segment.java @@ -18,14 +18,10 @@ package org.apache.carbondata.hadoop.internal.segment; import java.io.IOException; -import java.util.ArrayList; import java.util.List; import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapreduce.InputSplit; import org.apache.hadoop.mapreduce.JobContext; @@ -56,25 +52,6 @@ public abstract class Segment { } /** - * return all InputSplit of this segment, each file is a InputSplit - * @param job job context - * @return all InputSplit - * @throws IOException - */ - public List<InputSplit> getAllSplits(JobContext job) throws IOException { - List<InputSplit> result = new ArrayList<>(); - Path p = new Path(path); - FileSystem fs = p.getFileSystem(job.getConfiguration()); - - //TODO: filter out the hidden files - FileStatus[] files = fs.globStatus(p); - for (FileStatus file: files) { - // make split and add to result - } - return result; - } - - /** * get all files, implementation may use the input filter and index to prune files * @param job job context * @param filterResolver filter http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java ---------------------------------------------------------------------- diff --git a/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java b/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java index febca50..e95382c 100644 --- a/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java +++ b/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java @@ -196,12 +196,15 @@ public class CarbonDictionaryDecodeReadSupport<T> implements CarbonReadSupport<T if (carbonColumn.isDimension() && carbonColumn.getColumnSchema().getNumberOfChild() > 0) { childCarbonDimensions = ((CarbonDimension) carbonColumn).getListOfChildDimensions(); } - Writable[] arr = new Writable[objArray.length]; - for (int i = 0; i < objArray.length; i++) { - arr[i] = createWritableObject(objArray[i], childCarbonDimensions.get(i)); + if (null != childCarbonDimensions) { + Writable[] arr = new Writable[objArray.length]; + for (int i = 0; i < objArray.length; i++) { + + arr[i] = createWritableObject(objArray[i], childCarbonDimensions.get(i)); + } + return new ArrayWritable(Writable.class, arr); } - return new ArrayWritable(Writable.class, arr); } throw new IOException("DataType not supported in Carbondata"); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/integration/hive/src/main/java/org/apache/carbondata/hive/MapredCarbonInputFormat.java ---------------------------------------------------------------------- diff --git a/integration/hive/src/main/java/org/apache/carbondata/hive/MapredCarbonInputFormat.java b/integration/hive/src/main/java/org/apache/carbondata/hive/MapredCarbonInputFormat.java index 89a5ed6..d4cf480 100644 --- a/integration/hive/src/main/java/org/apache/carbondata/hive/MapredCarbonInputFormat.java +++ b/integration/hive/src/main/java/org/apache/carbondata/hive/MapredCarbonInputFormat.java @@ -77,13 +77,17 @@ public class MapredCarbonInputFormat extends CarbonTableInputFormat<ArrayWritabl } } } - AbsoluteTableIdentifier absoluteTableIdentifier = AbsoluteTableIdentifier - .from(validInputPath, getDatabaseName(configuration), getTableName(configuration)); - // read the schema file to get the absoluteTableIdentifier having the correct table id - // persisted in the schema - CarbonTable carbonTable = SchemaReader.readCarbonTableFromStore(absoluteTableIdentifier); - configuration.set(CARBON_TABLE, ObjectSerializationUtil.convertObjectToString(carbonTable)); - setTableInfo(configuration, carbonTable.getTableInfo()); + if (null != validInputPath) { + AbsoluteTableIdentifier absoluteTableIdentifier = AbsoluteTableIdentifier + .from(validInputPath, getDatabaseName(configuration), getTableName(configuration)); + // read the schema file to get the absoluteTableIdentifier having the correct table id + // persisted in the schema + CarbonTable carbonTable = SchemaReader.readCarbonTableFromStore(absoluteTableIdentifier); + configuration.set(CARBON_TABLE, ObjectSerializationUtil.convertObjectToString(carbonTable)); + setTableInfo(configuration, carbonTable.getTableInfo()); + } else { + throw new InvalidPathException("No input paths specified in job"); + } } private static CarbonTable getCarbonTable(Configuration configuration, String path) http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplitManager.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplitManager.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplitManager.java index 811393f..3a54b22 100755 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplitManager.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplitManager.java @@ -67,20 +67,23 @@ public class CarbondataSplitManager implements ConnectorSplitManager { getColumnConstraints(layoutHandle.getConstraint()); CarbonTableCacheModel cache = carbonTableReader.getCarbonCache(key); - Expression filters = PrestoFilterUtil.parseFilterExpression(layoutHandle.getConstraint()); - try { - List<CarbonLocalInputSplit> splits = carbonTableReader.getInputSplits2(cache, filters, layoutHandle.getConstraint()); - - ImmutableList.Builder<ConnectorSplit> cSplits = ImmutableList.builder(); - for (CarbonLocalInputSplit split : splits) { - cSplits.add(new CarbondataSplit(connectorId, tableHandle.getSchemaTableName(), - layoutHandle.getConstraint(), split, rebuildConstraints)); + if (null != cache) { + Expression filters = PrestoFilterUtil.parseFilterExpression(layoutHandle.getConstraint()); + try { + List<CarbonLocalInputSplit> splits = carbonTableReader.getInputSplits2(cache, filters, + layoutHandle.getConstraint()); + + ImmutableList.Builder<ConnectorSplit> cSplits = ImmutableList.builder(); + for (CarbonLocalInputSplit split : splits) { + cSplits.add(new CarbondataSplit(connectorId, tableHandle.getSchemaTableName(), + layoutHandle.getConstraint(), split, rebuildConstraints)); + } + return new FixedSplitSource(cSplits.build()); + } catch (Exception ex) { + throw new RuntimeException(ex.getMessage(), ex); } - return new FixedSplitSource(cSplits.build()); - } catch (Exception ex) { - throw new RuntimeException(ex.getMessage(), ex); } - + return null; } /** http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonLocalInputSplit.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonLocalInputSplit.java b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonLocalInputSplit.java index 3c42d0a..2c6a810 100755 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonLocalInputSplit.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonLocalInputSplit.java @@ -115,15 +115,21 @@ public class CarbonLocalInputSplit { } - public static CarbonInputSplit convertSplit(CarbonLocalInputSplit carbonLocalInputSplit) { + public static CarbonInputSplit convertSplit(CarbonLocalInputSplit carbonLocalInputSplit) { CarbonInputSplit inputSplit = new CarbonInputSplit(carbonLocalInputSplit.getSegmentId(), "0", new Path(carbonLocalInputSplit.getPath()), carbonLocalInputSplit.getStart(), carbonLocalInputSplit.getLength(), carbonLocalInputSplit.getLocations() .toArray(new String[carbonLocalInputSplit.getLocations().size()]), - carbonLocalInputSplit.getNumberOfBlocklets(), ColumnarFormatVersion.valueOf(carbonLocalInputSplit.getVersion()), + carbonLocalInputSplit.getNumberOfBlocklets(), + ColumnarFormatVersion.valueOf(carbonLocalInputSplit.getVersion()), carbonLocalInputSplit.getDeleteDeltaFiles()); Gson gson = new Gson(); - BlockletDetailInfo blockletDetailInfo = gson.fromJson(carbonLocalInputSplit.detailInfo, BlockletDetailInfo.class); + BlockletDetailInfo blockletDetailInfo = + gson.fromJson(carbonLocalInputSplit.detailInfo, BlockletDetailInfo.class); + + if (null == blockletDetailInfo) { + throw new RuntimeException("Could not read blocklet details"); + } try { blockletDetailInfo.readColumnSchema(blockletDetailInfo.getColumnSchemaBinary()); } catch (IOException e) { @@ -132,6 +138,4 @@ public class CarbonLocalInputSplit { inputSplit.setDetailInfo(blockletDetailInfo); return inputSplit; } - - } http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/integration/presto/src/main/java/org/apache/carbondata/presto/readers/BooleanStreamReader.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/readers/BooleanStreamReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/BooleanStreamReader.java index 4507425..0b7206b 100644 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/readers/BooleanStreamReader.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/BooleanStreamReader.java @@ -62,10 +62,8 @@ public class BooleanStreamReader extends AbstractStreamReader { } else { numberOfRows = streamData.length; builder = type.createBlockBuilder(new BlockBuilderStatus(), numberOfRows); - if (streamData != null) { - for (int i = 0; i < numberOfRows; i++) { - type.writeBoolean(builder, byteToBoolean(streamData[i])); - } + for (int i = 0; i < numberOfRows; i++) { + type.writeBoolean(builder, byteToBoolean(streamData[i])); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/integration/presto/src/main/java/org/apache/carbondata/presto/readers/DoubleStreamReader.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/readers/DoubleStreamReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/DoubleStreamReader.java index 23db769..3e7fc59 100644 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/readers/DoubleStreamReader.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/DoubleStreamReader.java @@ -59,7 +59,7 @@ public class DoubleStreamReader extends AbstractStreamReader { numberOfRows = batchSize; builder = type.createBlockBuilder(new BlockBuilderStatus(), numberOfRows); if (columnVector != null) { - if(isDictionary) { + if (isDictionary) { populateDictionaryVector(type, numberOfRows, builder); } else { if (columnVector.anyNullsSet()) { @@ -72,10 +72,8 @@ public class DoubleStreamReader extends AbstractStreamReader { } else { numberOfRows = streamData.length; builder = type.createBlockBuilder(new BlockBuilderStatus(), numberOfRows); - if (streamData != null) { - for (int i = 0; i < numberOfRows; i++) { - type.writeDouble(builder, (Double) streamData[i]); - } + for (int i = 0; i < numberOfRows; i++) { + type.writeDouble(builder, (Double) streamData[i]); } }
