[CARBONDATA-1308] Added tableProvider to supply carbonTable wherever needed
This closes #1208 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/d3a09e27 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/d3a09e27 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/d3a09e27 Branch: refs/heads/master Commit: d3a09e2790ec1d130feee78cdce5357a02c11628 Parents: e3f98fa Author: dhatchayani <[email protected]> Authored: Fri Jul 28 17:03:22 2017 +0530 Committer: Venkata Ramana G <[email protected]> Committed: Tue Aug 22 19:48:56 2017 +0530 ---------------------------------------------------------------------- .../dictionary/AbstractDictionaryCache.java | 9 ++-- .../dictionary/DictionaryCacheLoaderImpl.java | 44 ++++++++++-------- .../DictionaryColumnUniqueIdentifier.java | 12 ++++- .../IncrementalColumnDictionaryGenerator.java | 20 ++++++-- .../CarbonDictionaryMetadataReaderImpl.java | 20 ++++---- .../core/reader/CarbonDictionaryReaderImpl.java | 23 +++++----- .../CarbonDictionarySortIndexReaderImpl.java | 34 ++++++++------ .../executor/impl/AbstractQueryExecutor.java | 9 +++- .../core/scan/executor/util/QueryUtil.java | 24 ++++++---- .../scan/filter/FilterExpressionProcessor.java | 19 ++++---- .../core/scan/filter/FilterProcessor.java | 3 +- .../carbondata/core/scan/filter/FilterUtil.java | 31 +++++++++++-- .../core/scan/filter/SingleTableProvider.java | 40 ++++++++++++++++ .../core/scan/filter/TableProvider.java | 29 ++++++++++++ .../resolver/ConditionalFilterResolverImpl.java | 9 +++- .../filter/resolver/FilterResolverIntf.java | 3 +- .../resolver/LogicalFilterResolverImpl.java | 5 +- .../resolver/RowLevelFilterResolverImpl.java | 4 +- .../RowLevelRangeFilterResolverImpl.java | 3 +- .../metadata/FilterResolverMetadata.java | 11 +++++ .../TrueConditionalResolverImpl.java | 5 +- .../visitor/DictionaryColumnVisitor.java | 48 ++++++++++---------- .../visitor/RangeDictionaryColumnVisitor.java | 29 ++++++------ .../core/service/DictionaryService.java | 28 ++++++------ .../carbondata/core/service/PathService.java | 5 +- .../core/service/impl/DictionaryFactory.java | 48 ++++++++++---------- .../core/service/impl/PathFactory.java | 11 ++++- .../apache/carbondata/core/util/CarbonUtil.java | 3 +- .../core/writer/CarbonDictionaryWriterImpl.java | 37 ++++++++------- .../CarbonDictionarySortIndexWriterImpl.java | 27 ++++++----- .../dictionary/AbstractDictionaryCacheTest.java | 9 +++- .../DictionaryCacheLoaderImplTest.java | 16 +++++-- .../DictionaryColumnUniqueIdentifierTest.java | 6 +-- .../dictionary/ForwardDictionaryCacheTest.java | 7 ++- .../reader/CarbonDictionaryReaderImplTest.java | 8 +++- ...CarbonDictionarySortIndexReaderImplTest.java | 10 ++-- .../writer/CarbonDictionaryWriterImplTest.java | 13 ++++-- ...CarbonDictionarySortIndexWriterImplTest.java | 10 ++-- .../examples/GenerateDictionaryExample.scala | 4 +- .../carbondata/hadoop/CarbonInputFormat.java | 10 +++- .../hadoop/api/CarbonTableInputFormat.java | 14 ++++-- .../impl/DictionaryDecodeReadSupport.java | 4 +- .../hadoop/util/CarbonInputFormatUtil.java | 6 ++- .../hadoop/test/util/StoreCreator.java | 11 +++-- .../hive/CarbonDictionaryDecodeReadSupport.java | 4 +- .../hive/MapredCarbonInputFormat.java | 6 ++- .../presto/CarbondataRecordSetProvider.java | 7 ++- .../presto/impl/CarbonTableReader.java | 11 +++-- .../presto/CarbonDictionaryDecodeSupport.scala | 5 +- .../carbondata/spark/load/CarbonLoaderUtil.java | 3 +- .../spark/rdd/CarbonGlobalDictionaryRDD.scala | 17 +++++-- .../spark/tasks/DictionaryWriterTask.scala | 8 ++-- .../spark/tasks/SortIndexWriterTask.scala | 9 ++-- .../spark/util/GlobalDictionaryUtil.scala | 28 ++++++++++-- .../spark/sql/CarbonDictionaryDecoder.scala | 5 +- .../spark/util/DictionaryTestCaseUtil.scala | 4 +- ...GlobalDictionaryUtilConcurrentTestCase.scala | 10 +++- .../spark/sql/CarbonDictionaryDecoder.scala | 10 ++-- .../spark/util/DictionaryTestCaseUtil.scala | 4 +- .../processing/datatypes/PrimitiveDataType.java | 4 +- .../impl/DictionaryFieldConverterImpl.java | 4 +- .../carbondata/processing/StoreCreator.java | 9 ++-- 62 files changed, 592 insertions(+), 277 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/d3a09e27/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java index 6910888..4046364 100644 --- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java +++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java @@ -93,7 +93,7 @@ public abstract class AbstractDictionaryCache<K extends DictionaryColumnUniqueId DictionaryService dictService = CarbonCommonFactory.getDictionaryService(); CarbonDictionaryMetadataReader columnMetadataReaderImpl = dictService .getDictionaryMetadataReader(dictionaryColumnUniqueIdentifier.getCarbonTableIdentifier(), - dictionaryColumnUniqueIdentifier.getColumnIdentifier(), carbonStorePath); + dictionaryColumnUniqueIdentifier, carbonStorePath); CarbonDictionaryColumnMetaChunk carbonDictionaryColumnMetaChunk = null; // read metadata file @@ -121,7 +121,7 @@ public abstract class AbstractDictionaryCache<K extends DictionaryColumnUniqueId CarbonDictionaryMetadataReader columnMetadataReaderImpl = dictService .getDictionaryMetadataReader( dictionaryColumnUniqueIdentifier.getCarbonTableIdentifier(), - dictionaryColumnUniqueIdentifier.getColumnIdentifier(), carbonStorePath); + dictionaryColumnUniqueIdentifier, carbonStorePath); CarbonDictionaryColumnMetaChunk carbonDictionaryColumnMetaChunk = null; // read metadata file @@ -158,7 +158,8 @@ public abstract class AbstractDictionaryCache<K extends DictionaryColumnUniqueId DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) throws IOException { PathService pathService = CarbonCommonFactory.getPathService(); CarbonTablePath carbonTablePath = pathService.getCarbonTablePath(carbonStorePath, - dictionaryColumnUniqueIdentifier.getCarbonTableIdentifier()); + dictionaryColumnUniqueIdentifier.getCarbonTableIdentifier(), + dictionaryColumnUniqueIdentifier); String dictionaryFilePath = carbonTablePath.getDictionaryMetaFilePath( dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId()); FileFactory.FileType fileType = FileFactory.getFileType(dictionaryFilePath); @@ -280,7 +281,7 @@ public abstract class AbstractDictionaryCache<K extends DictionaryColumnUniqueId throws IOException { DictionaryCacheLoader dictionaryCacheLoader = new DictionaryCacheLoaderImpl(dictionaryColumnUniqueIdentifier.getCarbonTableIdentifier(), - carbonStorePath); + carbonStorePath, dictionaryColumnUniqueIdentifier); dictionaryCacheLoader .load(dictionaryInfo, dictionaryColumnUniqueIdentifier.getColumnIdentifier(), dictionaryChunkStartOffset, dictionaryChunkEndOffset, loadSortIndex); http://git-wip-us.apache.org/repos/asf/carbondata/blob/d3a09e27/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java index 5373ca8..e6a4022 100644 --- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java @@ -41,6 +41,8 @@ public class DictionaryCacheLoaderImpl implements DictionaryCacheLoader { */ private CarbonTableIdentifier carbonTableIdentifier; + private DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier; + /** * carbon store path */ @@ -51,9 +53,10 @@ public class DictionaryCacheLoaderImpl implements DictionaryCacheLoader { * @param carbonStorePath hdfs store path */ public DictionaryCacheLoaderImpl(CarbonTableIdentifier carbonTableIdentifier, - String carbonStorePath) { + String carbonStorePath, DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) { this.carbonTableIdentifier = carbonTableIdentifier; this.carbonStorePath = carbonStorePath; + this.dictionaryColumnUniqueIdentifier = dictionaryColumnUniqueIdentifier; } /** @@ -74,9 +77,10 @@ public class DictionaryCacheLoaderImpl implements DictionaryCacheLoader { long dictionaryChunkStartOffset, long dictionaryChunkEndOffset, boolean loadSortIndex) throws IOException { Iterator<byte[]> columnDictionaryChunkWrapper = - load(columnIdentifier, dictionaryChunkStartOffset, dictionaryChunkEndOffset); + load(dictionaryColumnUniqueIdentifier, dictionaryChunkStartOffset, + dictionaryChunkEndOffset); if (loadSortIndex) { - readSortIndexFile(dictionaryInfo, columnIdentifier); + readSortIndexFile(dictionaryInfo, dictionaryColumnUniqueIdentifier); } fillDictionaryValuesAndAddToDictionaryChunks(dictionaryInfo, columnDictionaryChunkWrapper); } @@ -118,15 +122,15 @@ public class DictionaryCacheLoaderImpl implements DictionaryCacheLoader { /** * This method will load the dictionary data between a given start and end offset * - * @param columnIdentifier column unique identifier + * @param dictionaryColumnUniqueIdentifier column unique identifier * @param startOffset start offset of dictionary file * @param endOffset end offset of dictionary file * @return iterator over dictionary values * @throws IOException */ - private Iterator<byte[]> load(ColumnIdentifier columnIdentifier, long startOffset, long endOffset) - throws IOException { - CarbonDictionaryReader dictionaryReader = getDictionaryReader(columnIdentifier); + private Iterator<byte[]> load(DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier, + long startOffset, long endOffset) throws IOException { + CarbonDictionaryReader dictionaryReader = getDictionaryReader(dictionaryColumnUniqueIdentifier); try { return dictionaryReader.read(startOffset, endOffset); } finally { @@ -138,12 +142,13 @@ public class DictionaryCacheLoaderImpl implements DictionaryCacheLoader { * This method will read the sort index file and load into memory * * @param dictionaryInfo - * @param columnIdentifier + * @param dictionaryColumnUniqueIdentifier * @throws IOException */ - private void readSortIndexFile(DictionaryInfo dictionaryInfo, ColumnIdentifier columnIdentifier) - throws IOException { - CarbonDictionarySortIndexReader sortIndexReader = getSortIndexReader(columnIdentifier); + private void readSortIndexFile(DictionaryInfo dictionaryInfo, + DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) throws IOException { + CarbonDictionarySortIndexReader sortIndexReader = + getSortIndexReader(dictionaryColumnUniqueIdentifier); try { dictionaryInfo.setSortOrderIndex(sortIndexReader.readSortIndex()); dictionaryInfo.setSortReverseOrderIndex(sortIndexReader.readInvertedSortIndex()); @@ -155,22 +160,25 @@ public class DictionaryCacheLoaderImpl implements DictionaryCacheLoader { /** * This method will create a dictionary reader instance to read the dictionary file * - * @param columnIdentifier unique column identifier + * @param dictionaryColumnUniqueIdentifier unique column identifier * @return carbon dictionary reader instance */ - private CarbonDictionaryReader getDictionaryReader(ColumnIdentifier columnIdentifier) { + private CarbonDictionaryReader getDictionaryReader( + DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) { DictionaryService dictService = CarbonCommonFactory.getDictionaryService(); - return dictService - .getDictionaryReader(carbonTableIdentifier, columnIdentifier, carbonStorePath); + return dictService.getDictionaryReader(carbonTableIdentifier, dictionaryColumnUniqueIdentifier, + carbonStorePath); } /** - * @param columnIdentifier unique column identifier + * @param dictionaryColumnUniqueIdentifier unique column identifier * @return sort index reader instance */ - private CarbonDictionarySortIndexReader getSortIndexReader(ColumnIdentifier columnIdentifier) { + private CarbonDictionarySortIndexReader getSortIndexReader( + DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) { DictionaryService dictService = CarbonCommonFactory.getDictionaryService(); return dictService - .getDictionarySortIndexReader(carbonTableIdentifier, columnIdentifier, carbonStorePath); + .getDictionarySortIndexReader(carbonTableIdentifier, dictionaryColumnUniqueIdentifier, + carbonStorePath); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/d3a09e27/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifier.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifier.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifier.java index f0c5c5a..f0182e4 100644 --- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifier.java +++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifier.java @@ -22,6 +22,7 @@ import java.io.Serializable; import org.apache.carbondata.core.metadata.CarbonTableIdentifier; import org.apache.carbondata.core.metadata.ColumnIdentifier; import org.apache.carbondata.core.metadata.datatype.DataType; +import org.apache.carbondata.core.util.path.CarbonTablePath; /** * dictionary column identifier which includes table identifier and column identifier @@ -38,6 +39,8 @@ public class DictionaryColumnUniqueIdentifier implements Serializable { */ private ColumnIdentifier columnIdentifier; + private transient CarbonTablePath carbonTablePath; + private DataType dataType; /** @@ -69,9 +72,12 @@ public class DictionaryColumnUniqueIdentifier implements Serializable { * @param dataType */ public DictionaryColumnUniqueIdentifier(CarbonTableIdentifier carbonTableIdentifier, - ColumnIdentifier columnIdentifier, DataType dataType) { + ColumnIdentifier columnIdentifier, DataType dataType, CarbonTablePath carbonTablePath) { this(carbonTableIdentifier, columnIdentifier); this.dataType = dataType; + if (null != carbonTablePath) { + this.carbonTablePath = carbonTablePath; + } } public DataType getDataType() { @@ -85,6 +91,10 @@ public class DictionaryColumnUniqueIdentifier implements Serializable { return carbonTableIdentifier; } + public CarbonTablePath getCarbonTablePath() { + return carbonTablePath; + } + /** * @return columnIdentifier */ http://git-wip-us.apache.org/repos/asf/carbondata/blob/d3a09e27/core/src/main/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGenerator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGenerator.java b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGenerator.java index 87c575f..20802be 100644 --- a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGenerator.java +++ b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGenerator.java @@ -42,6 +42,8 @@ import org.apache.carbondata.core.service.CarbonCommonFactory; import org.apache.carbondata.core.service.DictionaryService; import org.apache.carbondata.core.util.CarbonUtil; import org.apache.carbondata.core.util.DataTypeUtil; +import org.apache.carbondata.core.util.path.CarbonStorePath; +import org.apache.carbondata.core.util.path.CarbonTablePath; import org.apache.carbondata.core.writer.CarbonDictionaryWriter; import org.apache.carbondata.core.writer.sortindex.CarbonDictionarySortIndexWriter; import org.apache.carbondata.core.writer.sortindex.CarbonDictionarySortInfo; @@ -113,6 +115,8 @@ public class IncrementalColumnDictionaryGenerator implements BiDictionary<Intege // initialize params CarbonMetadata metadata = CarbonMetadata.getInstance(); CarbonTable carbonTable = metadata.getCarbonTable(tableUniqueName); + CarbonTablePath carbonTablePath = + CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier()); CarbonTableIdentifier tableIdentifier = carbonTable.getCarbonTableIdentifier(); ColumnIdentifier columnIdentifier = dimension.getColumnIdentifier(); String storePath = carbonTable.getStorePath(); @@ -120,7 +124,7 @@ public class IncrementalColumnDictionaryGenerator implements BiDictionary<Intege // create dictionary cache from dictionary File DictionaryColumnUniqueIdentifier identifier = new DictionaryColumnUniqueIdentifier(tableIdentifier, columnIdentifier, - columnIdentifier.getDataType()); + columnIdentifier.getDataType(), carbonTablePath); Boolean isDictExists = CarbonUtil.isFileExistsForGivenColumn(storePath, identifier); Dictionary dictionary = null; long t1 = System.currentTimeMillis(); @@ -134,7 +138,7 @@ public class IncrementalColumnDictionaryGenerator implements BiDictionary<Intege // write dictionary CarbonDictionaryWriter dictionaryWriter = null; dictionaryWriter = dictionaryService - .getDictionaryWriter(tableIdentifier, columnIdentifier, storePath); + .getDictionaryWriter(tableIdentifier, identifier, storePath); List<String> distinctValues = writeDictionary(dictionaryWriter, isDictExists); long dictWriteTime = System.currentTimeMillis() - t2; long t3 = System.currentTimeMillis(); @@ -215,14 +219,20 @@ public class IncrementalColumnDictionaryGenerator implements BiDictionary<Intege ColumnIdentifier columnIdentifier, String storePath) throws IOException { CarbonDictionarySortIndexWriter carbonDictionarySortIndexWriter = null; + CarbonTablePath carbonTablePath = CarbonStorePath + .getCarbonTablePath(storePath, tableIdentifier.getDatabaseName(), + tableIdentifier.getTableName()); + DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier = + new DictionaryColumnUniqueIdentifier(tableIdentifier, columnIdentifier, + columnIdentifier.getDataType(), carbonTablePath); try { CarbonDictionarySortInfoPreparator preparator = new CarbonDictionarySortInfoPreparator(); CarbonDictionarySortInfo dictionarySortInfo = preparator.getDictionarySortInfo(distinctValues, dictionary, dimension.getDataType()); - carbonDictionarySortIndexWriter = - dictionaryService.getDictionarySortIndexWriter(tableIdentifier, columnIdentifier, - storePath); + carbonDictionarySortIndexWriter = dictionaryService + .getDictionarySortIndexWriter(tableIdentifier, dictionaryColumnUniqueIdentifier, + storePath); carbonDictionarySortIndexWriter.writeSortIndex(dictionarySortInfo.getSortIndex()); carbonDictionarySortIndexWriter .writeInvertedSortIndex(dictionarySortInfo.getSortIndexInverted()); http://git-wip-us.apache.org/repos/asf/carbondata/blob/d3a09e27/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java index 9356974..e09fda0 100644 --- a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java @@ -21,9 +21,9 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier; import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.metadata.CarbonTableIdentifier; -import org.apache.carbondata.core.metadata.ColumnIdentifier; import org.apache.carbondata.core.service.CarbonCommonFactory; import org.apache.carbondata.core.service.PathService; import org.apache.carbondata.core.util.path.CarbonTablePath; @@ -49,7 +49,7 @@ public class CarbonDictionaryMetadataReaderImpl implements CarbonDictionaryMetad /** * column identifier */ - protected ColumnIdentifier columnIdentifier; + protected DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier; /** * dictionary metadata file path @@ -66,13 +66,14 @@ public class CarbonDictionaryMetadataReaderImpl implements CarbonDictionaryMetad * * @param storePath carbon dictionary meta data store path * @param carbonTableIdentifier table identifier which will give table name and database name - * @param columnIdentifier column unique identifier + * @param dictionaryColumnUniqueIdentifier column unique identifier */ public CarbonDictionaryMetadataReaderImpl(String storePath, - CarbonTableIdentifier carbonTableIdentifier, ColumnIdentifier columnIdentifier) { + CarbonTableIdentifier carbonTableIdentifier, + DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) { this.storePath = storePath; this.carbonTableIdentifier = carbonTableIdentifier; - this.columnIdentifier = columnIdentifier; + this.dictionaryColumnUniqueIdentifier = dictionaryColumnUniqueIdentifier; initFileLocation(); } @@ -169,10 +170,11 @@ public class CarbonDictionaryMetadataReaderImpl implements CarbonDictionaryMetad */ protected void initFileLocation() { PathService pathService = CarbonCommonFactory.getPathService(); - CarbonTablePath carbonTablePath = - pathService.getCarbonTablePath(this.storePath, carbonTableIdentifier); - this.columnDictionaryMetadataFilePath = - carbonTablePath.getDictionaryMetaFilePath(columnIdentifier.getColumnId()); + CarbonTablePath carbonTablePath = pathService + .getCarbonTablePath(this.storePath, carbonTableIdentifier, + dictionaryColumnUniqueIdentifier); + this.columnDictionaryMetadataFilePath = carbonTablePath.getDictionaryMetaFilePath( + dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId()); } /** http://git-wip-us.apache.org/repos/asf/carbondata/blob/d3a09e27/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImpl.java b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImpl.java index d93e3a5..0325cf6 100644 --- a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImpl.java @@ -24,8 +24,8 @@ import java.util.Iterator; import java.util.List; import org.apache.carbondata.core.cache.dictionary.ColumnDictionaryChunkIterator; +import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier; import org.apache.carbondata.core.metadata.CarbonTableIdentifier; -import org.apache.carbondata.core.metadata.ColumnIdentifier; import org.apache.carbondata.core.service.CarbonCommonFactory; import org.apache.carbondata.core.service.PathService; import org.apache.carbondata.core.util.path.CarbonTablePath; @@ -52,7 +52,7 @@ public class CarbonDictionaryReaderImpl implements CarbonDictionaryReader { /** * column name */ - protected ColumnIdentifier columnIdentifier; + protected DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier; /** * dictionary file path @@ -69,13 +69,13 @@ public class CarbonDictionaryReaderImpl implements CarbonDictionaryReader { * * @param storePath carbon dictionary data store path * @param carbonTableIdentifier table identifier which will give table name and database name - * @param columnIdentifier column unique identifier + * @param dictionaryColumnUniqueIdentifier column unique identifier */ - public CarbonDictionaryReaderImpl(String storePath, - CarbonTableIdentifier carbonTableIdentifier, ColumnIdentifier columnIdentifier) { + public CarbonDictionaryReaderImpl(String storePath, CarbonTableIdentifier carbonTableIdentifier, + DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) { this.storePath = storePath; this.carbonTableIdentifier = carbonTableIdentifier; - this.columnIdentifier = columnIdentifier; + this.dictionaryColumnUniqueIdentifier = dictionaryColumnUniqueIdentifier; initFileLocation(); } @@ -216,10 +216,11 @@ public class CarbonDictionaryReaderImpl implements CarbonDictionaryReader { */ protected void initFileLocation() { PathService pathService = CarbonCommonFactory.getPathService(); - CarbonTablePath carbonTablePath = pathService.getCarbonTablePath( - this.storePath, carbonTableIdentifier); - this.columnDictionaryFilePath = carbonTablePath - .getDictionaryFilePath(columnIdentifier.getColumnId()); + CarbonTablePath carbonTablePath = pathService + .getCarbonTablePath(this.storePath, carbonTableIdentifier, + dictionaryColumnUniqueIdentifier); + this.columnDictionaryFilePath = carbonTablePath.getDictionaryFilePath( + dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId()); } /** @@ -303,7 +304,7 @@ public class CarbonDictionaryReaderImpl implements CarbonDictionaryReader { */ protected CarbonDictionaryMetadataReader getDictionaryMetadataReader() { return new CarbonDictionaryMetadataReaderImpl(this.storePath, carbonTableIdentifier, - this.columnIdentifier); + this.dictionaryColumnUniqueIdentifier); } /** http://git-wip-us.apache.org/repos/asf/carbondata/blob/d3a09e27/core/src/main/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java b/core/src/main/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java index eeb8235..8c06434 100644 --- a/core/src/main/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java @@ -21,9 +21,9 @@ import java.util.List; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; +import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier; import org.apache.carbondata.core.datastore.impl.FileFactory; import org.apache.carbondata.core.metadata.CarbonTableIdentifier; -import org.apache.carbondata.core.metadata.ColumnIdentifier; import org.apache.carbondata.core.reader.CarbonDictionaryColumnMetaChunk; import org.apache.carbondata.core.reader.CarbonDictionaryMetadataReader; import org.apache.carbondata.core.reader.CarbonDictionaryMetadataReaderImpl; @@ -48,7 +48,7 @@ public class CarbonDictionarySortIndexReaderImpl implements CarbonDictionarySort /** * column name */ - protected ColumnIdentifier columnIdentifier; + protected DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier; /** * store location @@ -77,14 +77,16 @@ public class CarbonDictionarySortIndexReaderImpl implements CarbonDictionarySort private ThriftReader dictionarySortIndexThriftReader; /** - * @param carbonTableIdentifier Carbon Table identifier holding the database name and table name - * @param columnIdentifier column name - * @param carbonStorePath carbon store path + * @param carbonTableIdentifier Carbon Table identifier holding the database name + * and table name + * @param dictionaryColumnUniqueIdentifier column name + * @param carbonStorePath carbon store path */ public CarbonDictionarySortIndexReaderImpl(final CarbonTableIdentifier carbonTableIdentifier, - final ColumnIdentifier columnIdentifier, final String carbonStorePath) { + final DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier, + final String carbonStorePath) { this.carbonTableIdentifier = carbonTableIdentifier; - this.columnIdentifier = columnIdentifier; + this.dictionaryColumnUniqueIdentifier = dictionaryColumnUniqueIdentifier; this.carbonStorePath = carbonStorePath; } @@ -151,21 +153,23 @@ public class CarbonDictionarySortIndexReaderImpl implements CarbonDictionarySort protected void initPath() { PathService pathService = CarbonCommonFactory.getPathService(); - CarbonTablePath carbonTablePath = - pathService.getCarbonTablePath(carbonStorePath, carbonTableIdentifier); + CarbonTablePath carbonTablePath = pathService + .getCarbonTablePath(carbonStorePath, carbonTableIdentifier, + dictionaryColumnUniqueIdentifier); try { CarbonDictionaryColumnMetaChunk chunkMetaObjectForLastSegmentEntry = getChunkMetaObjectForLastSegmentEntry(); long dictOffset = chunkMetaObjectForLastSegmentEntry.getEnd_offset(); - this.sortIndexFilePath = - carbonTablePath.getSortIndexFilePath(columnIdentifier.getColumnId(), dictOffset); + this.sortIndexFilePath = carbonTablePath.getSortIndexFilePath( + dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId(), dictOffset); if (!FileFactory .isFileExist(this.sortIndexFilePath, FileFactory.getFileType(this.sortIndexFilePath))) { - this.sortIndexFilePath = - carbonTablePath.getSortIndexFilePath(columnIdentifier.getColumnId()); + this.sortIndexFilePath = carbonTablePath.getSortIndexFilePath( + dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId()); } } catch (IOException e) { - this.sortIndexFilePath = carbonTablePath.getSortIndexFilePath(columnIdentifier.getColumnId()); + this.sortIndexFilePath = carbonTablePath.getSortIndexFilePath( + dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId()); } } @@ -193,7 +197,7 @@ public class CarbonDictionarySortIndexReaderImpl implements CarbonDictionarySort */ protected CarbonDictionaryMetadataReader getDictionaryMetadataReader() { return new CarbonDictionaryMetadataReaderImpl(carbonStorePath, carbonTableIdentifier, - columnIdentifier); + dictionaryColumnUniqueIdentifier); } /** http://git-wip-us.apache.org/repos/asf/carbondata/blob/d3a09e27/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java index 05d0d8d..f159744 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java @@ -50,6 +50,7 @@ import org.apache.carbondata.core.memory.UnsafeMemoryManager; import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; import org.apache.carbondata.core.metadata.datatype.DataType; import org.apache.carbondata.core.metadata.encoder.Encoding; +import org.apache.carbondata.core.metadata.schema.table.CarbonTable; import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension; import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure; import org.apache.carbondata.core.scan.executor.QueryExecutor; @@ -58,6 +59,8 @@ import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo; import org.apache.carbondata.core.scan.executor.util.QueryUtil; import org.apache.carbondata.core.scan.executor.util.RestructureUtil; import org.apache.carbondata.core.scan.filter.FilterUtil; +import org.apache.carbondata.core.scan.filter.SingleTableProvider; +import org.apache.carbondata.core.scan.filter.TableProvider; import org.apache.carbondata.core.scan.model.QueryDimension; import org.apache.carbondata.core.scan.model.QueryMeasure; import org.apache.carbondata.core.scan.model.QueryModel; @@ -181,12 +184,16 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> { QueryUtil.getAllFilterDimensions(queryModel.getFilterExpressionResolverTree(), queryProperties.complexFilterDimension, queryProperties.filterMeasures); + CarbonTable carbonTable = queryModel.getTable(); + TableProvider tableProvider = new SingleTableProvider(carbonTable); + queryStatistic = new QueryStatistic(); // dictionary column unique column id to dictionary mapping // which will be used to get column actual data queryProperties.columnToDictionayMapping = QueryUtil .getDimensionDictionaryDetail(queryModel.getQueryDimension(), - queryProperties.complexFilterDimension, queryModel.getAbsoluteTableIdentifier()); + queryProperties.complexFilterDimension, queryModel.getAbsoluteTableIdentifier(), + tableProvider); queryStatistic .addStatistics(QueryStatisticsConstants.LOAD_DICTIONARY, System.currentTimeMillis()); queryProperties.queryStatisticsRecorder.recordStatistics(queryStatistic); http://git-wip-us.apache.org/repos/asf/carbondata/blob/d3a09e27/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java index ee04bb6..ee09793 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java @@ -54,12 +54,15 @@ import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo; import org.apache.carbondata.core.scan.expression.ColumnExpression; import org.apache.carbondata.core.scan.expression.Expression; import org.apache.carbondata.core.scan.filter.GenericQueryType; +import org.apache.carbondata.core.scan.filter.TableProvider; import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf; import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo; import org.apache.carbondata.core.scan.model.QueryDimension; import org.apache.carbondata.core.scan.model.QueryMeasure; import org.apache.carbondata.core.scan.model.QueryModel; import org.apache.carbondata.core.util.CarbonUtil; +import org.apache.carbondata.core.util.path.CarbonStorePath; +import org.apache.carbondata.core.util.path.CarbonTablePath; import org.apache.commons.lang3.ArrayUtils; @@ -273,7 +276,8 @@ public class QueryUtil { */ public static Map<String, Dictionary> getDimensionDictionaryDetail( List<QueryDimension> queryDimensions, Set<CarbonDimension> filterComplexDimensions, - AbsoluteTableIdentifier absoluteTableIdentifier) throws IOException { + AbsoluteTableIdentifier absoluteTableIdentifier, TableProvider tableProvider) + throws IOException { // to store dimension unique column id list, this is required as // dimension can be present in // query dimension, as well as some aggregation function will be applied @@ -307,7 +311,7 @@ public class QueryUtil { List<String> dictionaryColumnIdList = new ArrayList<String>(dictionaryDimensionFromQuery.size()); dictionaryColumnIdList.addAll(dictionaryDimensionFromQuery); - return getDictionaryMap(dictionaryColumnIdList, absoluteTableIdentifier); + return getDictionaryMap(dictionaryColumnIdList, absoluteTableIdentifier, tableProvider); } /** @@ -339,11 +343,12 @@ public class QueryUtil { * @throws IOException */ private static Map<String, Dictionary> getDictionaryMap(List<String> dictionaryColumnIdList, - AbsoluteTableIdentifier absoluteTableIdentifier) throws IOException { + AbsoluteTableIdentifier absoluteTableIdentifier, TableProvider tableProvider) + throws IOException { // this for dictionary unique identifier List<DictionaryColumnUniqueIdentifier> dictionaryColumnUniqueIdentifiers = getDictionaryColumnUniqueIdentifierList(dictionaryColumnIdList, - absoluteTableIdentifier.getCarbonTableIdentifier()); + absoluteTableIdentifier.getCarbonTableIdentifier(), tableProvider); CacheProvider cacheProvider = CacheProvider.getInstance(); Cache<DictionaryColumnUniqueIdentifier, Dictionary> forwardDictionaryCache = cacheProvider .createCache(CacheType.FORWARD_DICTIONARY, absoluteTableIdentifier.getStorePath()); @@ -367,9 +372,11 @@ public class QueryUtil { * @return */ private static List<DictionaryColumnUniqueIdentifier> getDictionaryColumnUniqueIdentifierList( - List<String> dictionaryColumnIdList, CarbonTableIdentifier carbonTableIdentifier) { - CarbonTable carbonTable = - CarbonMetadata.getInstance().getCarbonTable(carbonTableIdentifier.getTableUniqueName()); + List<String> dictionaryColumnIdList, CarbonTableIdentifier carbonTableIdentifier, + TableProvider tableProvider) throws IOException { + CarbonTable carbonTable = tableProvider.getCarbonTable(carbonTableIdentifier); + CarbonTablePath carbonTablePath = + CarbonStorePath.getCarbonTablePath(carbonTable.getStorePath(), carbonTableIdentifier); List<DictionaryColumnUniqueIdentifier> dictionaryColumnUniqueIdentifiers = new ArrayList<>(dictionaryColumnIdList.size()); for (String columnId : dictionaryColumnIdList) { @@ -380,7 +387,8 @@ public class QueryUtil { new DictionaryColumnUniqueIdentifier( carbonTableIdentifier, dimension.getColumnIdentifier(), - dimension.getDataType() + dimension.getDataType(), + carbonTablePath ) ); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/d3a09e27/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java index 9bc864d..cfcf112 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java @@ -80,9 +80,10 @@ public class FilterExpressionProcessor implements FilterProcessor { * @return a filter resolver tree */ public FilterResolverIntf getFilterResolver(Expression expressionTree, - AbsoluteTableIdentifier tableIdentifier) throws FilterUnsupportedException, IOException { + AbsoluteTableIdentifier tableIdentifier, TableProvider tableProvider) + throws FilterUnsupportedException, IOException { if (null != expressionTree && null != tableIdentifier) { - return getFilterResolvertree(expressionTree, tableIdentifier); + return getFilterResolvertree(expressionTree, tableIdentifier, tableProvider); } return null; } @@ -315,10 +316,11 @@ public class FilterExpressionProcessor implements FilterProcessor { * @return FilterResolverIntf type. */ private FilterResolverIntf getFilterResolvertree(Expression expressionTree, - AbsoluteTableIdentifier tableIdentifier) throws FilterUnsupportedException, IOException { + AbsoluteTableIdentifier tableIdentifier, TableProvider tableProvider) + throws FilterUnsupportedException, IOException { FilterResolverIntf filterEvaluatorTree = createFilterResolverTree(expressionTree, tableIdentifier); - traverseAndResolveTree(filterEvaluatorTree, tableIdentifier); + traverseAndResolveTree(filterEvaluatorTree, tableIdentifier, tableProvider); return filterEvaluatorTree; } @@ -332,13 +334,14 @@ public class FilterExpressionProcessor implements FilterProcessor { * @param tableIdentifier */ private void traverseAndResolveTree(FilterResolverIntf filterResolverTree, - AbsoluteTableIdentifier tableIdentifier) throws FilterUnsupportedException, IOException { + AbsoluteTableIdentifier tableIdentifier, TableProvider tableProvider) + throws FilterUnsupportedException, IOException { if (null == filterResolverTree) { return; } - traverseAndResolveTree(filterResolverTree.getLeft(), tableIdentifier); - filterResolverTree.resolve(tableIdentifier); - traverseAndResolveTree(filterResolverTree.getRight(), tableIdentifier); + traverseAndResolveTree(filterResolverTree.getLeft(), tableIdentifier, tableProvider); + filterResolverTree.resolve(tableIdentifier, tableProvider); + traverseAndResolveTree(filterResolverTree.getRight(), tableIdentifier, tableProvider); } /** http://git-wip-us.apache.org/repos/asf/carbondata/blob/d3a09e27/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterProcessor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterProcessor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterProcessor.java index 246166d..dc392ba 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterProcessor.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterProcessor.java @@ -41,7 +41,8 @@ public interface FilterProcessor { * @throws FilterUnsupportedException */ FilterResolverIntf getFilterResolver(Expression expressionTree, - AbsoluteTableIdentifier tableIdentifier) throws FilterUnsupportedException, IOException; + AbsoluteTableIdentifier tableIdentifier, TableProvider tableProvider) + throws FilterUnsupportedException, IOException; /** * This API is exposed inorder to get the required block reference node http://git-wip-us.apache.org/repos/asf/carbondata/blob/d3a09e27/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java index 6230771..b952b97 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java @@ -55,6 +55,7 @@ import org.apache.carbondata.core.keygenerator.KeyGenerator; import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; import org.apache.carbondata.core.metadata.datatype.DataType; import org.apache.carbondata.core.metadata.encoder.Encoding; +import org.apache.carbondata.core.metadata.schema.table.CarbonTable; import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension; import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure; import org.apache.carbondata.core.scan.expression.ColumnExpression; @@ -96,6 +97,8 @@ import org.apache.carbondata.core.util.DataTypeConverterImpl; import org.apache.carbondata.core.util.DataTypeUtil; import org.apache.carbondata.core.util.comparator.Comparator; import org.apache.carbondata.core.util.comparator.SerializableComparator; +import org.apache.carbondata.core.util.path.CarbonStorePath; +import org.apache.carbondata.core.util.path.CarbonTablePath; public final class FilterUtil { private static final LogService LOGGER = @@ -498,13 +501,15 @@ public final class FilterUtil { * @throws IOException */ public static ColumnFilterInfo getFilterValues(AbsoluteTableIdentifier tableIdentifier, - ColumnExpression columnExpression, List<String> evaluateResultList, boolean isIncludeFilter) + ColumnExpression columnExpression, List<String> evaluateResultList, boolean isIncludeFilter, + TableProvider tableProvider) throws IOException { Dictionary forwardDictionary = null; try { // Reading the dictionary value from cache. forwardDictionary = - getForwardDictionaryCache(tableIdentifier, columnExpression.getDimension()); + getForwardDictionaryCache(tableIdentifier, columnExpression.getDimension(), + tableProvider); return getFilterValues(columnExpression, evaluateResultList, forwardDictionary, isIncludeFilter); } finally { @@ -566,14 +571,15 @@ public final class FilterUtil { */ public static ColumnFilterInfo getFilterListForAllValues( AbsoluteTableIdentifier tableIdentifier, Expression expression, - final ColumnExpression columnExpression, boolean isIncludeFilter) + final ColumnExpression columnExpression, boolean isIncludeFilter, TableProvider tableProvider) throws IOException, FilterUnsupportedException { Dictionary forwardDictionary = null; List<String> evaluateResultListFinal = new ArrayList<String>(20); DictionaryChunksWrapper dictionaryWrapper = null; try { forwardDictionary = - getForwardDictionaryCache(tableIdentifier, columnExpression.getDimension()); + getForwardDictionaryCache(tableIdentifier, columnExpression.getDimension(), + tableProvider); dictionaryWrapper = forwardDictionary.getDictionaryChunks(); while (dictionaryWrapper.hasNext()) { byte[] columnVal = dictionaryWrapper.next(); @@ -1090,9 +1096,24 @@ public final class FilterUtil { */ public static Dictionary getForwardDictionaryCache(AbsoluteTableIdentifier tableIdentifier, CarbonDimension carbonDimension) throws IOException { + return getForwardDictionaryCache(tableIdentifier, carbonDimension, null); + } + + /** + * @param tableIdentifier + * @param carbonDimension + * @param tableProvider + * @return + */ + public static Dictionary getForwardDictionaryCache(AbsoluteTableIdentifier tableIdentifier, + CarbonDimension carbonDimension, TableProvider tableProvider) throws IOException { + CarbonTable carbonTable = + tableProvider.getCarbonTable(tableIdentifier.getCarbonTableIdentifier()); + CarbonTablePath carbonTablePath = + CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier()); DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier = new DictionaryColumnUniqueIdentifier(tableIdentifier.getCarbonTableIdentifier(), - carbonDimension.getColumnIdentifier(), carbonDimension.getDataType()); + carbonDimension.getColumnIdentifier(), carbonDimension.getDataType(), carbonTablePath); CacheProvider cacheProvider = CacheProvider.getInstance(); Cache<DictionaryColumnUniqueIdentifier, Dictionary> forwardDictionaryCache = cacheProvider.createCache(CacheType.FORWARD_DICTIONARY, tableIdentifier.getStorePath()); http://git-wip-us.apache.org/repos/asf/carbondata/blob/d3a09e27/core/src/main/java/org/apache/carbondata/core/scan/filter/SingleTableProvider.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/SingleTableProvider.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/SingleTableProvider.java new file mode 100644 index 0000000..88fc8a6 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/SingleTableProvider.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.carbondata.core.scan.filter; + +import java.io.IOException; + +import org.apache.carbondata.core.metadata.CarbonTableIdentifier; +import org.apache.carbondata.core.metadata.schema.table.CarbonTable; + +public class SingleTableProvider implements TableProvider { + + private CarbonTable carbonTable; + + public SingleTableProvider(CarbonTable carbonTable) { + this.carbonTable = carbonTable; + } + + @Override public CarbonTable getCarbonTable(CarbonTableIdentifier carbonTableIdentifier) + throws IOException { + if (carbonTable.getCarbonTableIdentifier().equals(carbonTableIdentifier)) { + return carbonTable; + } else { + throw new IOException("Carbon table does not exist with identifier " + carbonTableIdentifier); + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/carbondata/blob/d3a09e27/core/src/main/java/org/apache/carbondata/core/scan/filter/TableProvider.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/TableProvider.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/TableProvider.java new file mode 100644 index 0000000..d3fa15a --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/TableProvider.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.carbondata.core.scan.filter; + +import java.io.IOException; +import java.io.Serializable; + +import org.apache.carbondata.core.metadata.CarbonTableIdentifier; +import org.apache.carbondata.core.metadata.schema.table.CarbonTable; + +public interface TableProvider extends Serializable { + + CarbonTable getCarbonTable(CarbonTableIdentifier carbonTableIdentifier) throws IOException; + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/carbondata/blob/d3a09e27/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java index 8d3c366..6dd59fd 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java @@ -31,6 +31,7 @@ import org.apache.carbondata.core.scan.expression.conditional.ConditionalExpress import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException; import org.apache.carbondata.core.scan.expression.logical.RangeExpression; import org.apache.carbondata.core.scan.filter.FilterUtil; +import org.apache.carbondata.core.scan.filter.TableProvider; import org.apache.carbondata.core.scan.filter.intf.FilterExecuterType; import org.apache.carbondata.core.scan.filter.resolver.metadata.FilterResolverMetadata; import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo; @@ -70,7 +71,8 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf { * * @throws FilterUnsupportedException */ - @Override public void resolve(AbsoluteTableIdentifier absoluteTableIdentifier) + @Override public void resolve(AbsoluteTableIdentifier absoluteTableIdentifier, + TableProvider tableProvider) throws FilterUnsupportedException, IOException { FilterResolverMetadata metadata = new FilterResolverMetadata(); metadata.setTableIdentifier(absoluteTableIdentifier); @@ -83,6 +85,7 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf { metadata.setColumnExpression(columnExpression); metadata.setExpression(rightExp); metadata.setIncludeFilter(isIncludeFilter); + metadata.setTableProvider(tableProvider); // If imei=imei comes in filter condition then we need to // skip processing of right expression. // This flow has reached here assuming that this is a single @@ -118,6 +121,7 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf { metadata.setColumnExpression(columnExpression); metadata.setExpression(leftExp); metadata.setIncludeFilter(isIncludeFilter); + metadata.setTableProvider(tableProvider); if (columnExpression.getDataType().equals(DataType.TIMESTAMP) || columnExpression.getDataType().equals(DataType.DATE)) { isExpressionResolve = true; @@ -153,6 +157,7 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf { metadata.setColumnExpression(columnList.get(0)); metadata.setExpression(exp); metadata.setIncludeFilter(isIncludeFilter); + metadata.setTableProvider(tableProvider); if ((null != columnList.get(0).getDimension()) && ( !columnList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY) || columnList.get(0) .getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) @@ -169,7 +174,7 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf { == org.apache.carbondata.core.metadata.datatype.DataType.ARRAY))) { dimColResolvedFilterInfo.setFilterValues(FilterUtil .getFilterListForAllValues(absoluteTableIdentifier, exp, columnList.get(0), - isIncludeFilter)); + isIncludeFilter, tableProvider)); dimColResolvedFilterInfo.setColumnIndex(columnList.get(0).getDimension().getOrdinal()); dimColResolvedFilterInfo.setDimension(columnList.get(0).getDimension()); http://git-wip-us.apache.org/repos/asf/carbondata/blob/d3a09e27/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/FilterResolverIntf.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/FilterResolverIntf.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/FilterResolverIntf.java index 89a3890..a0810d5 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/FilterResolverIntf.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/FilterResolverIntf.java @@ -25,6 +25,7 @@ import org.apache.carbondata.core.datastore.block.SegmentProperties; import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; import org.apache.carbondata.core.scan.expression.Expression; import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException; +import org.apache.carbondata.core.scan.filter.TableProvider; import org.apache.carbondata.core.scan.filter.intf.FilterExecuterType; import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo; import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo; @@ -39,7 +40,7 @@ public interface FilterResolverIntf extends Serializable { * @throws IOException * @throws FilterUnsupportedException */ - void resolve(AbsoluteTableIdentifier absoluteTableIdentifier) + void resolve(AbsoluteTableIdentifier absoluteTableIdentifier, TableProvider tableProvider) throws IOException, FilterUnsupportedException; /** http://git-wip-us.apache.org/repos/asf/carbondata/blob/d3a09e27/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/LogicalFilterResolverImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/LogicalFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/LogicalFilterResolverImpl.java index 368cd9a..d530155 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/LogicalFilterResolverImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/LogicalFilterResolverImpl.java @@ -24,6 +24,7 @@ import org.apache.carbondata.core.datastore.block.SegmentProperties; import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; import org.apache.carbondata.core.scan.expression.BinaryExpression; import org.apache.carbondata.core.scan.expression.Expression; +import org.apache.carbondata.core.scan.filter.TableProvider; import org.apache.carbondata.core.scan.filter.intf.ExpressionType; import org.apache.carbondata.core.scan.filter.intf.FilterExecuterType; import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo; @@ -56,8 +57,10 @@ public class LogicalFilterResolverImpl implements FilterResolverIntf { * node for filter evaluation, so in this instance no implementation is required. * * @param absoluteTableIdentifier + * @param tableProvider */ - @Override public void resolve(AbsoluteTableIdentifier absoluteTableIdentifier) { + @Override public void resolve(AbsoluteTableIdentifier absoluteTableIdentifier, + TableProvider tableProvider) { } http://git-wip-us.apache.org/repos/asf/carbondata/blob/d3a09e27/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java index b399fd6..33bafb1 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java @@ -25,6 +25,7 @@ import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; import org.apache.carbondata.core.scan.expression.ColumnExpression; import org.apache.carbondata.core.scan.expression.Expression; import org.apache.carbondata.core.scan.expression.conditional.ConditionalExpression; +import org.apache.carbondata.core.scan.filter.TableProvider; import org.apache.carbondata.core.scan.filter.intf.FilterExecuterType; import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo; import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo; @@ -51,7 +52,8 @@ public class RowLevelFilterResolverImpl extends ConditionalFilterResolverImpl { * Method which will resolve the filter expression by converting the filter member * to its assigned dictionary values. */ - public void resolve(AbsoluteTableIdentifier absoluteTableIdentifier) { + public void resolve(AbsoluteTableIdentifier absoluteTableIdentifier, + TableProvider tableProvider) { DimColumnResolvedFilterInfo dimColumnEvaluatorInfo = null; MeasureColumnResolvedFilterInfo msrColumnEvalutorInfo = null; int index = 0; http://git-wip-us.apache.org/repos/asf/carbondata/blob/d3a09e27/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java index f64342f..3e27594 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java @@ -40,6 +40,7 @@ import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedExc import org.apache.carbondata.core.scan.expression.logical.BinaryLogicalExpression; import org.apache.carbondata.core.scan.filter.ColumnFilterInfo; import org.apache.carbondata.core.scan.filter.FilterUtil; +import org.apache.carbondata.core.scan.filter.TableProvider; import org.apache.carbondata.core.scan.filter.intf.FilterExecuterType; import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo; import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo; @@ -216,7 +217,7 @@ public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverIm * Method which will resolve the filter expression by converting the filter * member to its assigned dictionary values. */ - public void resolve(AbsoluteTableIdentifier absoluteTableIdentifier) + public void resolve(AbsoluteTableIdentifier absoluteTableIdentifier, TableProvider tableProvider) throws FilterUnsupportedException { DimColumnResolvedFilterInfo dimColumnEvaluatorInfo = null; MeasureColumnResolvedFilterInfo msrColumnEvalutorInfo = null; http://git-wip-us.apache.org/repos/asf/carbondata/blob/d3a09e27/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/metadata/FilterResolverMetadata.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/metadata/FilterResolverMetadata.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/metadata/FilterResolverMetadata.java index 1faaf86..8db80dc 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/metadata/FilterResolverMetadata.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/metadata/FilterResolverMetadata.java @@ -19,11 +19,22 @@ package org.apache.carbondata.core.scan.filter.resolver.metadata; import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; import org.apache.carbondata.core.scan.expression.ColumnExpression; import org.apache.carbondata.core.scan.expression.Expression; +import org.apache.carbondata.core.scan.filter.TableProvider; public class FilterResolverMetadata { private AbsoluteTableIdentifier tableIdentifier; private Expression expression; private ColumnExpression columnExpression; + + public TableProvider getTableProvider() { + return tableProvider; + } + + public void setTableProvider(TableProvider tableProvider) { + this.tableProvider = tableProvider; + } + + private TableProvider tableProvider; private boolean isIncludeFilter; public AbsoluteTableIdentifier getTableIdentifier() { http://git-wip-us.apache.org/repos/asf/carbondata/blob/d3a09e27/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/TrueConditionalResolverImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/TrueConditionalResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/TrueConditionalResolverImpl.java index c9d180f..d403ef4 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/TrueConditionalResolverImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/TrueConditionalResolverImpl.java @@ -19,6 +19,7 @@ package org.apache.carbondata.core.scan.filter.resolver.resolverinfo; import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; import org.apache.carbondata.core.scan.expression.Expression; +import org.apache.carbondata.core.scan.filter.TableProvider; import org.apache.carbondata.core.scan.filter.intf.FilterExecuterType; import org.apache.carbondata.core.scan.filter.resolver.ConditionalFilterResolverImpl; @@ -32,8 +33,8 @@ public class TrueConditionalResolverImpl extends ConditionalFilterResolverImpl { super(exp, isExpressionResolve, isIncludeFilter, tableIdentifier, false); } - @Override - public void resolve(AbsoluteTableIdentifier absoluteTableIdentifier) { + @Override public void resolve(AbsoluteTableIdentifier absoluteTableIdentifier, + TableProvider tableProvider) { } /** http://git-wip-us.apache.org/repos/asf/carbondata/blob/d3a09e27/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java index 2fc7503..451ce45 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java @@ -45,32 +45,30 @@ public class DictionaryColumnVisitor implements ResolvedFilterInfoVisitorIntf { public void populateFilterResolvedInfo(ColumnResolvedFilterInfo visitableObj, FilterResolverMetadata metadata) throws FilterUnsupportedException, IOException { - if (visitableObj instanceof DimColumnResolvedFilterInfo) { - DimColumnResolvedFilterInfo resolveDimension = (DimColumnResolvedFilterInfo) visitableObj; - ColumnFilterInfo resolvedFilterObject = null; - List<String> evaluateResultListFinal; - try { - evaluateResultListFinal = metadata.getExpression().evaluate(null).getListAsString(); - } catch (FilterIllegalMemberException e) { - throw new FilterUnsupportedException(e); - } - resolvedFilterObject = FilterUtil - .getFilterValues(metadata.getTableIdentifier(), metadata.getColumnExpression(), - evaluateResultListFinal, metadata.isIncludeFilter()); - if (!metadata.isIncludeFilter() && null != resolvedFilterObject) { - // Adding default surrogate key of null member inorder to not display the same while - // displaying the report as per hive compatibility. - // first check of surrogate key for null value is already added then - // no need to add again otherwise result will be wrong in case of exclude filter - // this is because two times it will flip the same bit - if (!resolvedFilterObject.getFilterList() - .contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY)) { - resolvedFilterObject.getFilterList() - .add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY); - } - Collections.sort(resolvedFilterObject.getFilterList()); + DimColumnResolvedFilterInfo resolveDimension = (DimColumnResolvedFilterInfo) visitableObj; + ColumnFilterInfo resolvedFilterObject = null; + List<String> evaluateResultListFinal; + try { + evaluateResultListFinal = metadata.getExpression().evaluate(null).getListAsString(); + } catch (FilterIllegalMemberException e) { + throw new FilterUnsupportedException(e); + } + resolvedFilterObject = FilterUtil + .getFilterValues(metadata.getTableIdentifier(), metadata.getColumnExpression(), + evaluateResultListFinal, metadata.isIncludeFilter(), metadata.getTableProvider()); + if (!metadata.isIncludeFilter() && null != resolvedFilterObject) { + // Adding default surrogate key of null member inorder to not display the same while + // displaying the report as per hive compatibility. + // first check of surrogate key for null value is already added then + // no need to add again otherwise result will be wrong in case of exclude filter + // this is because two times it will flip the same bit + if (!resolvedFilterObject.getFilterList() + .contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY)) { + resolvedFilterObject.getFilterList() + .add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY); } - resolveDimension.setFilterValues(resolvedFilterObject); + Collections.sort(resolvedFilterObject.getFilterList()); } + resolveDimension.setFilterValues(resolvedFilterObject); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/d3a09e27/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDictionaryColumnVisitor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDictionaryColumnVisitor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDictionaryColumnVisitor.java index 3d16cd2..5ce818d 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDictionaryColumnVisitor.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDictionaryColumnVisitor.java @@ -44,22 +44,21 @@ public class RangeDictionaryColumnVisitor extends DictionaryColumnVisitor */ public void populateFilterResolvedInfo(ColumnResolvedFilterInfo visitableObj, FilterResolverMetadata metadata) throws FilterUnsupportedException, IOException { - if (visitableObj instanceof DimColumnResolvedFilterInfo) { - DimColumnResolvedFilterInfo resolveDimension = (DimColumnResolvedFilterInfo) visitableObj; - ColumnFilterInfo resolvedFilterObject = null; - List<String> evaluateResultListFinal; - resolvedFilterObject = FilterUtil - .getFilterListForAllValues(metadata.getTableIdentifier(), metadata.getExpression(), - metadata.getColumnExpression(), metadata.isIncludeFilter()); + DimColumnResolvedFilterInfo resolveDimension = (DimColumnResolvedFilterInfo) visitableObj; + ColumnFilterInfo resolvedFilterObject = null; + List<String> evaluateResultListFinal; + resolvedFilterObject = FilterUtil + .getFilterListForAllValues(metadata.getTableIdentifier(), metadata.getExpression(), + metadata.getColumnExpression(), metadata.isIncludeFilter(), + metadata.getTableProvider()); - if (!metadata.isIncludeFilter() && null != resolvedFilterObject) { - // Adding default surrogate key of null member inorder to not display the same while - // displaying the report as per hive compatibility. - resolvedFilterObject.getFilterList() - .add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY); - Collections.sort(resolvedFilterObject.getFilterList()); - } - resolveDimension.setFilterValues(resolvedFilterObject); + if (!metadata.isIncludeFilter() && null != resolvedFilterObject) { + // Adding default surrogate key of null member inorder to not display the same while + // displaying the report as per hive compatibility. + resolvedFilterObject.getFilterList() + .add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY); + Collections.sort(resolvedFilterObject.getFilterList()); } + resolveDimension.setFilterValues(resolvedFilterObject); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/d3a09e27/core/src/main/java/org/apache/carbondata/core/service/DictionaryService.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/service/DictionaryService.java b/core/src/main/java/org/apache/carbondata/core/service/DictionaryService.java index ef941a5..672aa83 100644 --- a/core/src/main/java/org/apache/carbondata/core/service/DictionaryService.java +++ b/core/src/main/java/org/apache/carbondata/core/service/DictionaryService.java @@ -16,8 +16,8 @@ */ package org.apache.carbondata.core.service; +import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier; import org.apache.carbondata.core.metadata.CarbonTableIdentifier; -import org.apache.carbondata.core.metadata.ColumnIdentifier; import org.apache.carbondata.core.reader.CarbonDictionaryMetadataReader; import org.apache.carbondata.core.reader.CarbonDictionaryReader; import org.apache.carbondata.core.reader.sortindex.CarbonDictionarySortIndexReader; @@ -33,58 +33,58 @@ public interface DictionaryService { * get dictionary writer * * @param carbonTableIdentifier - * @param columnIdentifier + * @param dictionaryColumnUniqueIdentifier * @param carbonStorePath * @return */ CarbonDictionaryWriter getDictionaryWriter(CarbonTableIdentifier carbonTableIdentifier, - ColumnIdentifier columnIdentifier, String carbonStorePath); + DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier, String carbonStorePath); /** * get dictionary sort index writer * * @param carbonTableIdentifier - * @param columnIdentifier + * @param dictionaryColumnUniqueIdentifier * @param carbonStorePath * @return */ CarbonDictionarySortIndexWriter getDictionarySortIndexWriter( - CarbonTableIdentifier carbonTableIdentifier, ColumnIdentifier columnIdentifier, - String carbonStorePath); + CarbonTableIdentifier carbonTableIdentifier, + DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier, String carbonStorePath); /** * get dictionary metadata reader * * @param carbonTableIdentifier - * @param columnIdentifier + * @param dictionaryColumnUniqueIdentifier * @param carbonStorePath * @return */ CarbonDictionaryMetadataReader getDictionaryMetadataReader( - CarbonTableIdentifier carbonTableIdentifier, ColumnIdentifier columnIdentifier, - String carbonStorePath); + CarbonTableIdentifier carbonTableIdentifier, + DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier, String carbonStorePath); /** * get dictionary reader * * @param carbonTableIdentifier - * @param columnIdentifier + * @param dictionaryColumnUniqueIdentifier * @param carbonStorePath * @return */ CarbonDictionaryReader getDictionaryReader(CarbonTableIdentifier carbonTableIdentifier, - ColumnIdentifier columnIdentifier, String carbonStorePath); + DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier, String carbonStorePath); /** * get dictionary sort index reader * * @param carbonTableIdentifier - * @param columnIdentifier + * @param dictionaryColumnUniqueIdentifier * @param carbonStorePath * @return */ CarbonDictionarySortIndexReader getDictionarySortIndexReader( - CarbonTableIdentifier carbonTableIdentifier, ColumnIdentifier columnIdentifier, - String carbonStorePath); + CarbonTableIdentifier carbonTableIdentifier, + DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier, String carbonStorePath); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/d3a09e27/core/src/main/java/org/apache/carbondata/core/service/PathService.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/service/PathService.java b/core/src/main/java/org/apache/carbondata/core/service/PathService.java index 3528d0e..9735217 100644 --- a/core/src/main/java/org/apache/carbondata/core/service/PathService.java +++ b/core/src/main/java/org/apache/carbondata/core/service/PathService.java @@ -16,6 +16,7 @@ */ package org.apache.carbondata.core.service; +import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier; import org.apache.carbondata.core.metadata.CarbonTableIdentifier; import org.apache.carbondata.core.util.path.CarbonTablePath; @@ -27,7 +28,9 @@ public interface PathService { /** * @param storeLocation * @param tableIdentifier + * @param dictionaryColumnUniqueIdentifier * @return store path related to tables */ - CarbonTablePath getCarbonTablePath(String storeLocation, CarbonTableIdentifier tableIdentifier); + CarbonTablePath getCarbonTablePath(String storeLocation, CarbonTableIdentifier tableIdentifier, + DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/d3a09e27/core/src/main/java/org/apache/carbondata/core/service/impl/DictionaryFactory.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/service/impl/DictionaryFactory.java b/core/src/main/java/org/apache/carbondata/core/service/impl/DictionaryFactory.java index 8d6969c..1dc22ba 100644 --- a/core/src/main/java/org/apache/carbondata/core/service/impl/DictionaryFactory.java +++ b/core/src/main/java/org/apache/carbondata/core/service/impl/DictionaryFactory.java @@ -16,8 +16,8 @@ */ package org.apache.carbondata.core.service.impl; +import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier; import org.apache.carbondata.core.metadata.CarbonTableIdentifier; -import org.apache.carbondata.core.metadata.ColumnIdentifier; import org.apache.carbondata.core.reader.CarbonDictionaryMetadataReader; import org.apache.carbondata.core.reader.CarbonDictionaryMetadataReaderImpl; import org.apache.carbondata.core.reader.CarbonDictionaryReader; @@ -41,73 +41,75 @@ public class DictionaryFactory implements DictionaryService { * get dictionary writer * * @param carbonTableIdentifier - * @param columnIdentifier + * @param dictionaryColumnUniqueIdentifier * @param carbonStorePath * @return */ @Override public CarbonDictionaryWriter getDictionaryWriter( - CarbonTableIdentifier carbonTableIdentifier, ColumnIdentifier columnIdentifier, - String carbonStorePath) { - return new CarbonDictionaryWriterImpl(carbonStorePath, carbonTableIdentifier, columnIdentifier); + CarbonTableIdentifier carbonTableIdentifier, + DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier, String carbonStorePath) { + return new CarbonDictionaryWriterImpl(carbonStorePath, carbonTableIdentifier, + dictionaryColumnUniqueIdentifier); } /** * get dictionary sort index writer * * @param carbonTableIdentifier - * @param columnIdentifier + * @param dictionaryColumnUniqueIdentifier * @param carbonStorePath * @return */ @Override public CarbonDictionarySortIndexWriter getDictionarySortIndexWriter( - CarbonTableIdentifier carbonTableIdentifier, ColumnIdentifier columnIdentifier, - String carbonStorePath) { - return new CarbonDictionarySortIndexWriterImpl(carbonTableIdentifier, columnIdentifier, - carbonStorePath); + CarbonTableIdentifier carbonTableIdentifier, + DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier, String carbonStorePath) { + return new CarbonDictionarySortIndexWriterImpl(carbonTableIdentifier, + dictionaryColumnUniqueIdentifier, carbonStorePath); } /** * get dictionary metadata reader * * @param carbonTableIdentifier - * @param columnIdentifier + * @param dictionaryColumnUniqueIdentifier * @param carbonStorePath * @return */ @Override public CarbonDictionaryMetadataReader getDictionaryMetadataReader( - CarbonTableIdentifier carbonTableIdentifier, ColumnIdentifier columnIdentifier, - String carbonStorePath) { + CarbonTableIdentifier carbonTableIdentifier, + DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier, String carbonStorePath) { return new CarbonDictionaryMetadataReaderImpl(carbonStorePath, carbonTableIdentifier, - columnIdentifier); + dictionaryColumnUniqueIdentifier); } /** * get dictionary reader * * @param carbonTableIdentifier - * @param columnIdentifier + * @param dictionaryColumnUniqueIdentifier * @param carbonStorePath * @return */ @Override public CarbonDictionaryReader getDictionaryReader( - CarbonTableIdentifier carbonTableIdentifier, ColumnIdentifier columnIdentifier, - String carbonStorePath) { - return new CarbonDictionaryReaderImpl(carbonStorePath, carbonTableIdentifier, columnIdentifier); + CarbonTableIdentifier carbonTableIdentifier, + DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier, String carbonStorePath) { + return new CarbonDictionaryReaderImpl(carbonStorePath, carbonTableIdentifier, + dictionaryColumnUniqueIdentifier); } /** * get dictionary sort index reader * * @param carbonTableIdentifier - * @param columnIdentifier + * @param dictionaryColumnUniqueIdentifier * @param carbonStorePath * @return */ @Override public CarbonDictionarySortIndexReader getDictionarySortIndexReader( - CarbonTableIdentifier carbonTableIdentifier, ColumnIdentifier columnIdentifier, - String carbonStorePath) { - return new CarbonDictionarySortIndexReaderImpl(carbonTableIdentifier, columnIdentifier, - carbonStorePath); + CarbonTableIdentifier carbonTableIdentifier, + DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier, String carbonStorePath) { + return new CarbonDictionarySortIndexReaderImpl(carbonTableIdentifier, + dictionaryColumnUniqueIdentifier, carbonStorePath); } public static DictionaryService getInstance() { http://git-wip-us.apache.org/repos/asf/carbondata/blob/d3a09e27/core/src/main/java/org/apache/carbondata/core/service/impl/PathFactory.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/service/impl/PathFactory.java b/core/src/main/java/org/apache/carbondata/core/service/impl/PathFactory.java index b3c7d61..e7e9ebe 100644 --- a/core/src/main/java/org/apache/carbondata/core/service/impl/PathFactory.java +++ b/core/src/main/java/org/apache/carbondata/core/service/impl/PathFactory.java @@ -16,6 +16,7 @@ */ package org.apache.carbondata.core.service.impl; +import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier; import org.apache.carbondata.core.metadata.CarbonTableIdentifier; import org.apache.carbondata.core.service.PathService; import org.apache.carbondata.core.util.path.CarbonStorePath; @@ -31,10 +32,16 @@ public class PathFactory implements PathService { /** * @param storeLocation * @param tableIdentifier + * @param * @return store path related to tables */ - @Override public CarbonTablePath getCarbonTablePath( - String storeLocation, CarbonTableIdentifier tableIdentifier) { + @Override public CarbonTablePath getCarbonTablePath(String storeLocation, + CarbonTableIdentifier tableIdentifier, + DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) { + if (null != dictionaryColumnUniqueIdentifier && null != dictionaryColumnUniqueIdentifier + .getCarbonTablePath()) { + return dictionaryColumnUniqueIdentifier.getCarbonTablePath(); + } return CarbonStorePath.getCarbonTablePath(storeLocation, tableIdentifier); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/d3a09e27/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java index 3bf837b..1db1cca 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java @@ -1576,7 +1576,8 @@ public final class CarbonUtil { DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) { PathService pathService = CarbonCommonFactory.getPathService(); CarbonTablePath carbonTablePath = pathService.getCarbonTablePath(carbonStorePath, - dictionaryColumnUniqueIdentifier.getCarbonTableIdentifier()); + dictionaryColumnUniqueIdentifier.getCarbonTableIdentifier(), + dictionaryColumnUniqueIdentifier); String dictionaryFilePath = carbonTablePath.getDictionaryFilePath( dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId());
