[CARBONDATA-1870] Add dictionary path support to carbondata This closes #1617
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/29dc3028 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/29dc3028 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/29dc3028 Branch: refs/heads/master Commit: 29dc30280d37e39306006c288546206db06d1bf4 Parents: 2304303 Author: manishgupta88 <[email protected]> Authored: Tue Dec 5 20:39:43 2017 +0530 Committer: Venkata Ramana G <[email protected]> Committed: Thu Dec 7 22:39:39 2017 +0530 ---------------------------------------------------------------------- .../dictionary/AbstractDictionaryCache.java | 9 +- .../DictionaryColumnUniqueIdentifier.java | 95 +++++++++++++++----- .../core/constants/CarbonCommonConstants.java | 4 + .../IncrementalColumnDictionaryGenerator.java | 11 +-- .../CarbonDictionaryMetadataReaderImpl.java | 11 +-- .../core/reader/CarbonDictionaryReaderImpl.java | 10 +-- .../CarbonDictionarySortIndexReaderImpl.java | 16 +--- .../core/scan/executor/util/QueryUtil.java | 13 +-- .../carbondata/core/scan/filter/FilterUtil.java | 18 ++-- .../apache/carbondata/core/util/CarbonUtil.java | 13 +-- .../core/util/path/CarbonTablePath.java | 40 ++++++++- .../core/writer/CarbonDictionaryWriterImpl.java | 13 +-- .../CarbonDictionarySortIndexWriterImpl.java | 26 ++---- .../dictionary/AbstractDictionaryCacheTest.java | 6 +- .../DictionaryCacheLoaderImplTest.java | 3 +- .../dictionary/ForwardDictionaryCacheTest.java | 3 +- .../reader/CarbonDictionaryReaderImplTest.java | 3 +- ...CarbonDictionarySortIndexReaderImplTest.java | 3 +- .../writer/CarbonDictionaryWriterImplTest.java | 3 +- ...CarbonDictionarySortIndexWriterImplTest.java | 3 +- .../carbondata/hadoop/CarbonRecordReader.java | 3 +- .../hadoop/readsupport/CarbonReadSupport.java | 7 +- .../impl/DictionaryDecodeReadSupport.java | 15 ++-- .../readsupport/impl/RawDataReadSupport.java | 5 +- .../streaming/CarbonStreamInputFormat.java | 7 +- .../hadoop/test/util/StoreCreator.java | 8 +- .../hive/CarbonDictionaryDecodeReadSupport.java | 13 +-- .../carbondata/hive/CarbonHiveRecordReader.java | 2 +- .../carbondata/presto/CarbondataRecordSet.java | 2 +- .../CarbonDictionaryDecodeReadSupport.scala | 12 +-- .../presto/util/CarbonDataStoreCreator.scala | 11 +-- .../spark/rdd/CarbonGlobalDictionaryRDD.scala | 3 +- .../spark/util/GlobalDictionaryUtil.scala | 9 +- .../readsupport/SparkRowReadSupportImpl.java | 4 +- .../spark/sql/CarbonDictionaryDecoder.scala | 54 ++++++----- .../spark/util/DictionaryTestCaseUtil.scala | 4 +- .../processing/datatypes/PrimitiveDataType.java | 4 +- .../converter/impl/FieldEncoderFactory.java | 6 +- .../processing/util/CarbonLoaderUtil.java | 3 +- .../carbondata/processing/StoreCreator.java | 7 +- 40 files changed, 244 insertions(+), 238 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/29dc3028/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java index d527e68..598d00e 100644 --- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java +++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java @@ -30,10 +30,8 @@ import org.apache.carbondata.core.reader.CarbonDictionaryColumnMetaChunk; import org.apache.carbondata.core.reader.CarbonDictionaryMetadataReader; import org.apache.carbondata.core.service.CarbonCommonFactory; import org.apache.carbondata.core.service.DictionaryService; -import org.apache.carbondata.core.service.PathService; import org.apache.carbondata.core.util.CarbonProperties; import org.apache.carbondata.core.util.ObjectSizeCalculator; -import org.apache.carbondata.core.util.path.CarbonTablePath; /** * Abstract class which implements methods common to reverse and forward dictionary cache @@ -141,12 +139,7 @@ public abstract class AbstractDictionaryCache<K extends DictionaryColumnUniqueId */ private CarbonFile getDictionaryMetaCarbonFile( DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) throws IOException { - PathService pathService = CarbonCommonFactory.getPathService(); - CarbonTablePath carbonTablePath = pathService - .getCarbonTablePath(dictionaryColumnUniqueIdentifier.getAbsoluteCarbonTableIdentifier(), - dictionaryColumnUniqueIdentifier); - String dictionaryFilePath = carbonTablePath.getDictionaryMetaFilePath( - dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId()); + String dictionaryFilePath = dictionaryColumnUniqueIdentifier.getDictionaryFilePath(); FileFactory.FileType fileType = FileFactory.getFileType(dictionaryFilePath); CarbonFile dictFile = FileFactory.getCarbonFile(dictionaryFilePath, fileType); // When rename table triggered parallely with select query, dictionary files may not exist http://git-wip-us.apache.org/repos/asf/carbondata/blob/29dc3028/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifier.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifier.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifier.java index 1c733b4..f5652ac 100644 --- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifier.java +++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifier.java @@ -22,6 +22,7 @@ import java.io.Serializable; import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; import org.apache.carbondata.core.metadata.ColumnIdentifier; import org.apache.carbondata.core.metadata.datatype.DataType; +import org.apache.carbondata.core.util.path.CarbonStorePath; import org.apache.carbondata.core.util.path.CarbonTablePath; /** @@ -34,7 +35,7 @@ public class DictionaryColumnUniqueIdentifier implements Serializable { /** * table fully qualified name */ - private AbsoluteTableIdentifier absoluteTableIdentifier; + private AbsoluteTableIdentifier dictionarySourceAbsoluteTableIdentifier; /** * unique column id @@ -45,40 +46,53 @@ public class DictionaryColumnUniqueIdentifier implements Serializable { private DataType dataType; + private String dictionaryLocation; + /** * Will be used in case of reverse dictionary cache which will be used * in case of data loading. * - * @param absoluteTableIdentifier + * @param dictionarySourceAbsoluteTableIdentifier * @param columnIdentifier */ - public DictionaryColumnUniqueIdentifier(AbsoluteTableIdentifier absoluteTableIdentifier, + public DictionaryColumnUniqueIdentifier( + AbsoluteTableIdentifier dictionarySourceAbsoluteTableIdentifier, ColumnIdentifier columnIdentifier) { - if (absoluteTableIdentifier == null) { + if (dictionarySourceAbsoluteTableIdentifier == null) { throw new IllegalArgumentException("carbonTableIdentifier is null"); } if (columnIdentifier == null) { throw new IllegalArgumentException("columnIdentifier is null"); } - this.absoluteTableIdentifier = absoluteTableIdentifier; + this.dictionarySourceAbsoluteTableIdentifier = dictionarySourceAbsoluteTableIdentifier; this.columnIdentifier = columnIdentifier; this.dataType = columnIdentifier.getDataType(); + this.dictionaryLocation = + CarbonStorePath.getCarbonTablePath(dictionarySourceAbsoluteTableIdentifier) + .getMetadataDirectoryPath(); } /** * Will be used in case of forward dictionary cache in case * of query execution. * - * @param absoluteTableIdentifier + * @param dictionarySourceAbsoluteTableIdentifier * @param columnIdentifier * @param dataType */ - public DictionaryColumnUniqueIdentifier(AbsoluteTableIdentifier absoluteTableIdentifier, - ColumnIdentifier columnIdentifier, DataType dataType, CarbonTablePath carbonTablePath) { - this(absoluteTableIdentifier, columnIdentifier); + public DictionaryColumnUniqueIdentifier( + AbsoluteTableIdentifier dictionarySourceAbsoluteTableIdentifier, + ColumnIdentifier columnIdentifier, DataType dataType) { + this(dictionarySourceAbsoluteTableIdentifier, columnIdentifier); this.dataType = dataType; - if (null != carbonTablePath) { - this.carbonTablePath = carbonTablePath; + } + + public DictionaryColumnUniqueIdentifier( + AbsoluteTableIdentifier dictionarySourceAbsoluteTableIdentifier, + ColumnIdentifier columnIdentifier, DataType dataType, String dictionaryLocation) { + this(dictionarySourceAbsoluteTableIdentifier, columnIdentifier, dataType); + if (null != dictionaryLocation) { + this.dictionaryLocation = dictionaryLocation; } } @@ -86,14 +100,10 @@ public class DictionaryColumnUniqueIdentifier implements Serializable { return dataType; } - /** - * @return table identifier - */ - public AbsoluteTableIdentifier getAbsoluteCarbonTableIdentifier() { - return absoluteTableIdentifier; - } - public CarbonTablePath getCarbonTablePath() { + if (null == carbonTablePath) { + carbonTablePath = CarbonStorePath.getCarbonTablePath(dictionarySourceAbsoluteTableIdentifier); + } return carbonTablePath; } @@ -105,18 +115,57 @@ public class DictionaryColumnUniqueIdentifier implements Serializable { } /** + * @return dictionary file path + */ + public String getDictionaryFilePath() { + return getCarbonTablePath() + .getDictionaryFilePath(dictionaryLocation, columnIdentifier.getColumnId()); + } + + /** + * @return dictionary metadata file path + */ + public String getDictionaryMetaFilePath() { + return getCarbonTablePath() + .getDictionaryMetaFilePath(dictionaryLocation, columnIdentifier.getColumnId()); + } + + /** + * @return sort index file path + */ + public String getSortIndexFilePath() { + return getCarbonTablePath() + .getSortIndexFilePath(dictionaryLocation, columnIdentifier.getColumnId()); + } + + /** + * @param offset + * @return sort index file path with given offset + */ + public String getSortIndexFilePath(long offset) { + return getCarbonTablePath() + .getSortIndexFilePath(dictionaryLocation, columnIdentifier.getColumnId(), offset); + } + + /** * overridden equals method * * @param other * @return */ @Override public boolean equals(Object other) { - if (this == other) return true; - if (other == null || getClass() != other.getClass()) return false; + if (this == other) { + return true; + } + if (other == null || getClass() != other.getClass()) { + return false; + } DictionaryColumnUniqueIdentifier that = (DictionaryColumnUniqueIdentifier) other; - if (!absoluteTableIdentifier.equals(that.absoluteTableIdentifier)) return false; + if (!dictionarySourceAbsoluteTableIdentifier + .equals(that.dictionarySourceAbsoluteTableIdentifier)) { + return false; + } return columnIdentifier.equals(that.columnIdentifier); - } /** @@ -125,7 +174,7 @@ public class DictionaryColumnUniqueIdentifier implements Serializable { * @return */ @Override public int hashCode() { - int result = absoluteTableIdentifier.hashCode(); + int result = dictionarySourceAbsoluteTableIdentifier.hashCode(); result = 31 * result + columnIdentifier.hashCode(); return result; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/29dc3028/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java index 5fb08a3..4ab9919 100644 --- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java +++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java @@ -859,6 +859,10 @@ public final class CarbonCommonConstants { public static final String COLUMN_GROUPS = "column_groups"; public static final String DICTIONARY_EXCLUDE = "dictionary_exclude"; public static final String DICTIONARY_INCLUDE = "dictionary_include"; + /** + * key for dictionary path + */ + public static final String DICTIONARY_PATH = "dictionary_path"; public static final String SORT_COLUMNS = "sort_columns"; public static final String PARTITION_TYPE = "partition_type"; public static final String NUM_PARTITIONS = "num_partitions"; http://git-wip-us.apache.org/repos/asf/carbondata/blob/29dc3028/core/src/main/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGenerator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGenerator.java b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGenerator.java index a129671..5eeeae4 100644 --- a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGenerator.java +++ b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGenerator.java @@ -41,8 +41,6 @@ import org.apache.carbondata.core.service.CarbonCommonFactory; import org.apache.carbondata.core.service.DictionaryService; import org.apache.carbondata.core.util.CarbonUtil; import org.apache.carbondata.core.util.DataTypeUtil; -import org.apache.carbondata.core.util.path.CarbonStorePath; -import org.apache.carbondata.core.util.path.CarbonTablePath; import org.apache.carbondata.core.writer.CarbonDictionaryWriter; import org.apache.carbondata.core.writer.sortindex.CarbonDictionarySortIndexWriter; import org.apache.carbondata.core.writer.sortindex.CarbonDictionarySortInfo; @@ -116,15 +114,13 @@ public class IncrementalColumnDictionaryGenerator implements BiDictionary<Intege @Override public void writeDictionaryData() throws IOException { // initialize params - CarbonTablePath carbonTablePath = - CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier()); AbsoluteTableIdentifier absoluteTableIdentifier = carbonTable.getAbsoluteTableIdentifier(); ColumnIdentifier columnIdentifier = dimension.getColumnIdentifier(); DictionaryService dictionaryService = CarbonCommonFactory.getDictionaryService(); // create dictionary cache from dictionary File DictionaryColumnUniqueIdentifier identifier = - new DictionaryColumnUniqueIdentifier(absoluteTableIdentifier, columnIdentifier, - columnIdentifier.getDataType(), carbonTablePath); + new DictionaryColumnUniqueIdentifier(absoluteTableIdentifier, columnIdentifier, + columnIdentifier.getDataType()); Boolean isDictExists = CarbonUtil.isFileExistsForGivenColumn(identifier); Dictionary dictionary = null; long t1 = System.currentTimeMillis(); @@ -216,10 +212,9 @@ public class IncrementalColumnDictionaryGenerator implements BiDictionary<Intege AbsoluteTableIdentifier absoluteTableIdentifier, ColumnIdentifier columnIdentifier) throws IOException { CarbonDictionarySortIndexWriter carbonDictionarySortIndexWriter = null; - CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier); DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier = new DictionaryColumnUniqueIdentifier(absoluteTableIdentifier, columnIdentifier, - columnIdentifier.getDataType(), carbonTablePath); + columnIdentifier.getDataType()); try { CarbonDictionarySortInfoPreparator preparator = new CarbonDictionarySortInfoPreparator(); CarbonDictionarySortInfo dictionarySortInfo = http://git-wip-us.apache.org/repos/asf/carbondata/blob/29dc3028/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java index fa3c1eb..6439b36 100644 --- a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java @@ -23,9 +23,6 @@ import java.util.List; import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier; import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.service.CarbonCommonFactory; -import org.apache.carbondata.core.service.PathService; -import org.apache.carbondata.core.util.path.CarbonTablePath; import org.apache.carbondata.format.ColumnDictionaryChunkMeta; import org.apache.thrift.TBase; @@ -153,12 +150,8 @@ public class CarbonDictionaryMetadataReaderImpl implements CarbonDictionaryMetad * This method will form the path for dictionary metadata file for a given column */ protected void initFileLocation() { - PathService pathService = CarbonCommonFactory.getPathService(); - CarbonTablePath carbonTablePath = pathService - .getCarbonTablePath(dictionaryColumnUniqueIdentifier.getAbsoluteCarbonTableIdentifier(), - dictionaryColumnUniqueIdentifier); - this.columnDictionaryMetadataFilePath = carbonTablePath.getDictionaryMetaFilePath( - dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId()); + this.columnDictionaryMetadataFilePath = + dictionaryColumnUniqueIdentifier.getDictionaryMetaFilePath(); } /** http://git-wip-us.apache.org/repos/asf/carbondata/blob/29dc3028/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImpl.java b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImpl.java index 539ee16..c66b9fb 100644 --- a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImpl.java @@ -25,9 +25,6 @@ import java.util.List; import org.apache.carbondata.core.cache.dictionary.ColumnDictionaryChunkIterator; import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier; -import org.apache.carbondata.core.service.CarbonCommonFactory; -import org.apache.carbondata.core.service.PathService; -import org.apache.carbondata.core.util.path.CarbonTablePath; import org.apache.carbondata.format.ColumnDictionaryChunk; import org.apache.thrift.TBase; @@ -200,12 +197,7 @@ public class CarbonDictionaryReaderImpl implements CarbonDictionaryReader { * This method will form the path for dictionary file for a given column */ protected void initFileLocation() { - PathService pathService = CarbonCommonFactory.getPathService(); - CarbonTablePath carbonTablePath = pathService - .getCarbonTablePath(dictionaryColumnUniqueIdentifier.getAbsoluteCarbonTableIdentifier(), - dictionaryColumnUniqueIdentifier); - this.columnDictionaryFilePath = carbonTablePath.getDictionaryFilePath( - dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId()); + this.columnDictionaryFilePath = dictionaryColumnUniqueIdentifier.getDictionaryFilePath(); } /** http://git-wip-us.apache.org/repos/asf/carbondata/blob/29dc3028/core/src/main/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java b/core/src/main/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java index 12426f4..439a93e 100644 --- a/core/src/main/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java @@ -27,9 +27,6 @@ import org.apache.carbondata.core.reader.CarbonDictionaryColumnMetaChunk; import org.apache.carbondata.core.reader.CarbonDictionaryMetadataReader; import org.apache.carbondata.core.reader.CarbonDictionaryMetadataReaderImpl; import org.apache.carbondata.core.reader.ThriftReader; -import org.apache.carbondata.core.service.CarbonCommonFactory; -import org.apache.carbondata.core.service.PathService; -import org.apache.carbondata.core.util.path.CarbonTablePath; import org.apache.carbondata.format.ColumnSortInfo; import org.apache.thrift.TBase; @@ -135,24 +132,17 @@ public class CarbonDictionarySortIndexReaderImpl implements CarbonDictionarySort } protected void initPath() { - PathService pathService = CarbonCommonFactory.getPathService(); - CarbonTablePath carbonTablePath = pathService - .getCarbonTablePath(dictionaryColumnUniqueIdentifier.getAbsoluteCarbonTableIdentifier(), - dictionaryColumnUniqueIdentifier); try { CarbonDictionaryColumnMetaChunk chunkMetaObjectForLastSegmentEntry = getChunkMetaObjectForLastSegmentEntry(); long dictOffset = chunkMetaObjectForLastSegmentEntry.getEnd_offset(); - this.sortIndexFilePath = carbonTablePath.getSortIndexFilePath( - dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId(), dictOffset); + this.sortIndexFilePath = dictionaryColumnUniqueIdentifier.getSortIndexFilePath(dictOffset); if (!FileFactory .isFileExist(this.sortIndexFilePath, FileFactory.getFileType(this.sortIndexFilePath))) { - this.sortIndexFilePath = carbonTablePath.getSortIndexFilePath( - dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId()); + this.sortIndexFilePath = dictionaryColumnUniqueIdentifier.getSortIndexFilePath(); } } catch (IOException e) { - this.sortIndexFilePath = carbonTablePath.getSortIndexFilePath( - dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId()); + this.sortIndexFilePath = dictionaryColumnUniqueIdentifier.getSortIndexFilePath(); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/29dc3028/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java index b5c56dc..b33ab8d 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java @@ -385,23 +385,24 @@ public class QueryUtil { CarbonDimension dimension = CarbonMetadata.getInstance() .getCarbonDimensionBasedOnColIdentifier(carbonTable, columnId); if (dimension != null) { - AbsoluteTableIdentifier newCarbonTableIdentifier; + AbsoluteTableIdentifier dictionarySourceAbsoluteTableIdentifier; ColumnIdentifier columnIdentifier; if (null != dimension.getColumnSchema().getParentColumnTableRelations() && !dimension .getColumnSchema().getParentColumnTableRelations().isEmpty()) { - newCarbonTableIdentifier = getTableIdentifierForColumn(dimension, + dictionarySourceAbsoluteTableIdentifier = getTableIdentifierForColumn(dimension, carbonTable.getAbsoluteTableIdentifier()); columnIdentifier = new ColumnIdentifier( dimension.getColumnSchema().getParentColumnTableRelations().get(0).getColumnId(), dimension.getColumnProperties(), dimension.getDataType()); } else { - newCarbonTableIdentifier = carbonTable.getAbsoluteTableIdentifier(); + dictionarySourceAbsoluteTableIdentifier = carbonTable.getAbsoluteTableIdentifier(); columnIdentifier = dimension.getColumnIdentifier(); } + String dictionaryPath = carbonTable.getTableInfo().getFactTable().getTableProperties() + .get(CarbonCommonConstants.DICTIONARY_PATH); dictionaryColumnUniqueIdentifiers.add( - new DictionaryColumnUniqueIdentifier(newCarbonTableIdentifier, columnIdentifier, - dimension.getDataType(), - CarbonStorePath.getCarbonTablePath(newCarbonTableIdentifier))); + new DictionaryColumnUniqueIdentifier(dictionarySourceAbsoluteTableIdentifier, + columnIdentifier, dimension.getDataType(), dictionaryPath)); } } return dictionaryColumnUniqueIdentifiers; http://git-wip-us.apache.org/repos/asf/carbondata/blob/29dc3028/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java index 16b3b1f..a8fa8ec 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java @@ -101,8 +101,6 @@ import org.apache.carbondata.core.util.DataTypeConverterImpl; import org.apache.carbondata.core.util.DataTypeUtil; import org.apache.carbondata.core.util.comparator.Comparator; import org.apache.carbondata.core.util.comparator.SerializableComparator; -import org.apache.carbondata.core.util.path.CarbonStorePath; -import org.apache.carbondata.core.util.path.CarbonTablePath; import org.apache.commons.lang.ArrayUtils; import org.roaringbitmap.RoaringBitmap; @@ -1240,18 +1238,18 @@ public final class FilterUtil { * @return */ public static Dictionary getForwardDictionaryCache( - AbsoluteTableIdentifier absoluteTableIdentifier, + AbsoluteTableIdentifier dictionarySourceAbsoluteTableIdentifier, CarbonDimension carbonDimension, TableProvider tableProvider) throws IOException { - CarbonTablePath carbonTablePath = null; + String dictionaryPath = null; if (null != tableProvider) { - CarbonTable carbonTable = - tableProvider.getCarbonTable(absoluteTableIdentifier.getCarbonTableIdentifier()); - carbonTablePath = - CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier()); + CarbonTable carbonTable = tableProvider + .getCarbonTable(dictionarySourceAbsoluteTableIdentifier.getCarbonTableIdentifier()); + dictionaryPath = carbonTable.getTableInfo().getFactTable().getTableProperties() + .get(CarbonCommonConstants.DICTIONARY_PATH); } DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier = - new DictionaryColumnUniqueIdentifier(absoluteTableIdentifier, - carbonDimension.getColumnIdentifier(), carbonDimension.getDataType(), carbonTablePath); + new DictionaryColumnUniqueIdentifier(dictionarySourceAbsoluteTableIdentifier, + carbonDimension.getColumnIdentifier(), carbonDimension.getDataType(), dictionaryPath); CacheProvider cacheProvider = CacheProvider.getInstance(); Cache<DictionaryColumnUniqueIdentifier, Dictionary> forwardDictionaryCache = cacheProvider.createCache(CacheType.FORWARD_DICTIONARY); http://git-wip-us.apache.org/repos/asf/carbondata/blob/29dc3028/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java index 1e8bc1f..ab85684 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java @@ -76,8 +76,6 @@ import org.apache.carbondata.core.mutate.UpdateVO; import org.apache.carbondata.core.reader.ThriftReader; import org.apache.carbondata.core.reader.ThriftReader.TBaseCreator; import org.apache.carbondata.core.scan.model.QueryDimension; -import org.apache.carbondata.core.service.CarbonCommonFactory; -import org.apache.carbondata.core.service.PathService; import org.apache.carbondata.core.statusmanager.LoadMetadataDetails; import org.apache.carbondata.core.statusmanager.SegmentStatus; import org.apache.carbondata.core.statusmanager.SegmentStatusManager; @@ -1636,14 +1634,9 @@ public final class CarbonUtil { */ public static boolean isFileExistsForGivenColumn( DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) { - PathService pathService = CarbonCommonFactory.getPathService(); - CarbonTablePath carbonTablePath = pathService - .getCarbonTablePath(dictionaryColumnUniqueIdentifier.getAbsoluteCarbonTableIdentifier(), - dictionaryColumnUniqueIdentifier); - String dictionaryFilePath = carbonTablePath.getDictionaryFilePath( - dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId()); - String dictionaryMetadataFilePath = carbonTablePath.getDictionaryMetaFilePath( - dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId()); + String dictionaryFilePath = dictionaryColumnUniqueIdentifier.getDictionaryFilePath(); + String dictionaryMetadataFilePath = + dictionaryColumnUniqueIdentifier.getDictionaryMetaFilePath(); // check if both dictionary and its metadata file exists for a given column return isFileExists(dictionaryFilePath) && isFileExists(dictionaryMetadataFilePath); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/29dc3028/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java index 172b71d..0f85b64 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java +++ b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java @@ -139,6 +139,15 @@ public class CarbonTablePath extends Path { } /** + * @param dictionaryPath + * @param columnId unique column identifier + * @return absolute path of dictionary file + */ + public String getDictionaryFilePath(String dictionaryPath, String columnId) { + return dictionaryPath + File.separator + getDictionaryFileName(columnId); + } + + /** * This method will return the metadata directory location for a table * * @return @@ -163,6 +172,15 @@ public class CarbonTablePath extends Path { } /** + * @param dictionaryPath + * @param columnId unique column identifier + * @return absolute path of dictionary file + */ + public String getDictionaryMetaFilePath(String dictionaryPath, String columnId) { + return dictionaryPath + File.separator + columnId + DICTIONARY_META_EXT; + } + + /** * @param columnId unique column identifier * @return absolute path of sort index file */ @@ -171,6 +189,15 @@ public class CarbonTablePath extends Path { } /** + * @param dictionaryPath + * @param columnId unique column identifier + * @return absolute path of dictionary file + */ + public String getSortIndexFilePath(String dictionaryPath, String columnId) { + return dictionaryPath + File.separator + columnId + SORT_INDEX_EXT; + } + + /** * * @param columnId * @param dictOffset @@ -181,6 +208,16 @@ public class CarbonTablePath extends Path { } /** + * @param dictionaryPath + * @param columnId unique column identifier + * @param dictOffset + * @return absolute path of dictionary file + */ + public String getSortIndexFilePath(String dictionaryPath, String columnId, long dictOffset) { + return dictionaryPath + File.separator + columnId + "_" + dictOffset + SORT_INDEX_EXT; + } + + /** * @return absolute path of schema file */ public String getSchemaFilePath() { @@ -526,7 +563,8 @@ public class CarbonTablePath extends Path { * @param columnUniqueId columnunique id * @return sort index carbon files */ - public CarbonFile[] getSortIndexFiles(CarbonFile sortIndexDir, final String columnUniqueId) { + public static CarbonFile[] getSortIndexFiles(CarbonFile sortIndexDir, + final String columnUniqueId) { return sortIndexDir.listFiles(new CarbonFileFilter() { @Override public boolean accept(CarbonFile file) { return file.getName().startsWith(columnUniqueId) && file.getName().endsWith(SORT_INDEX_EXT); http://git-wip-us.apache.org/repos/asf/carbondata/blob/29dc3028/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java index 6e867bb..4014cb4 100644 --- a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java @@ -32,10 +32,7 @@ import org.apache.carbondata.core.datastore.impl.FileFactory; import org.apache.carbondata.core.reader.CarbonDictionaryColumnMetaChunk; import org.apache.carbondata.core.reader.CarbonDictionaryMetadataReader; import org.apache.carbondata.core.reader.CarbonDictionaryMetadataReaderImpl; -import org.apache.carbondata.core.service.CarbonCommonFactory; -import org.apache.carbondata.core.service.PathService; import org.apache.carbondata.core.util.CarbonUtil; -import org.apache.carbondata.core.util.path.CarbonTablePath; import org.apache.carbondata.core.util.path.HDFSLeaseUtils; import org.apache.carbondata.format.ColumnDictionaryChunk; import org.apache.carbondata.format.ColumnDictionaryChunkMeta; @@ -238,14 +235,8 @@ public class CarbonDictionaryWriterImpl implements CarbonDictionaryWriter { } protected void initPaths() { - PathService pathService = CarbonCommonFactory.getPathService(); - CarbonTablePath carbonTablePath = pathService - .getCarbonTablePath(dictionaryColumnUniqueIdentifier.getAbsoluteCarbonTableIdentifier(), - dictionaryColumnUniqueIdentifier); - this.dictionaryFilePath = carbonTablePath.getDictionaryFilePath( - dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId()); - this.dictionaryMetaFilePath = carbonTablePath.getDictionaryMetaFilePath( - dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId()); + this.dictionaryFilePath = dictionaryColumnUniqueIdentifier.getDictionaryFilePath(); + this.dictionaryMetaFilePath = dictionaryColumnUniqueIdentifier.getDictionaryMetaFilePath(); } /** http://git-wip-us.apache.org/repos/asf/carbondata/blob/29dc3028/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java b/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java index 840b25d..f25081d 100644 --- a/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java @@ -27,8 +27,6 @@ import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentif import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.datastore.filesystem.CarbonFile; import org.apache.carbondata.core.datastore.impl.FileFactory; -import org.apache.carbondata.core.service.CarbonCommonFactory; -import org.apache.carbondata.core.service.PathService; import org.apache.carbondata.core.util.CarbonProperties; import org.apache.carbondata.core.util.CarbonUtil; import org.apache.carbondata.core.util.path.CarbonTablePath; @@ -134,28 +132,20 @@ public class CarbonDictionarySortIndexWriterImpl implements CarbonDictionarySort } protected void initPath() { - PathService pathService = CarbonCommonFactory.getPathService(); - CarbonTablePath carbonTablePath = pathService - .getCarbonTablePath(dictionaryColumnUniqueIdentifier.getAbsoluteCarbonTableIdentifier(), - dictionaryColumnUniqueIdentifier); - String dictionaryPath = carbonTablePath.getDictionaryFilePath( - dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId()); - long dictOffset = CarbonUtil.getFileSize(dictionaryPath); - this.sortIndexFilePath = carbonTablePath - .getSortIndexFilePath(dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId(), - dictOffset); - cleanUpOldSortIndex(carbonTablePath, dictionaryPath); + String dictionaryFilePath = dictionaryColumnUniqueIdentifier.getDictionaryFilePath(); + long dictOffset = CarbonUtil.getFileSize(dictionaryFilePath); + this.sortIndexFilePath = dictionaryColumnUniqueIdentifier.getSortIndexFilePath(dictOffset); + cleanUpOldSortIndex(dictionaryFilePath); } /** * It cleans up old unused sortindex file * - * @param carbonTablePath + * @param dictPath */ - protected void cleanUpOldSortIndex(CarbonTablePath carbonTablePath, String dictPath) { - CarbonFile dictFile = - FileFactory.getCarbonFile(dictPath, FileFactory.getFileType(dictPath)); - CarbonFile[] files = carbonTablePath.getSortIndexFiles(dictFile.getParentFile(), + protected void cleanUpOldSortIndex(String dictPath) { + CarbonFile dictFile = FileFactory.getCarbonFile(dictPath, FileFactory.getFileType(dictPath)); + CarbonFile[] files = CarbonTablePath.getSortIndexFiles(dictFile.getParentFile(), dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId()); int maxTime; try { http://git-wip-us.apache.org/repos/asf/carbondata/blob/29dc3028/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java index 16c09fd..31e44a2 100644 --- a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java +++ b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java @@ -108,8 +108,7 @@ public class AbstractDictionaryCacheTest { String columnId) { ColumnIdentifier columnIdentifier = new ColumnIdentifier(columnId, null, DataTypes.STRING); return new DictionaryColumnUniqueIdentifier(absoluteTableIdentifier, columnIdentifier, - DataTypes.STRING, - CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier)); + DataTypes.STRING); } /** @@ -132,8 +131,7 @@ public class AbstractDictionaryCacheTest { ColumnIdentifier columnIdentifier = new ColumnIdentifier(columnId, null, null); DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier = new DictionaryColumnUniqueIdentifier(absoluteTableIdentifier, columnIdentifier, - columnIdentifier.getDataType(), - CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier)); + columnIdentifier.getDataType()); CarbonDictionaryWriter carbonDictionaryWriter = new CarbonDictionaryWriterImpl(dictionaryColumnUniqueIdentifier); CarbonTablePath carbonTablePath = http://git-wip-us.apache.org/repos/asf/carbondata/blob/29dc3028/core/src/test/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImplTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImplTest.java b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImplTest.java index 7b4a076..83e9217 100644 --- a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImplTest.java +++ b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImplTest.java @@ -57,8 +57,7 @@ public class DictionaryCacheLoaderImplTest { columnIdentifier = new ColumnIdentifier("1", columnProperties, DataTypes.STRING); dictionaryColumnUniqueIdentifier = new DictionaryColumnUniqueIdentifier(absoluteTableIdentifier, columnIdentifier, - columnIdentifier.getDataType(), CarbonStorePath.getCarbonTablePath("/tmp", - carbonTableIdentifier)); + columnIdentifier.getDataType()); dictionaryCacheLoader = new DictionaryCacheLoaderImpl(dictionaryColumnUniqueIdentifier); dictionaryInfo = new ColumnDictionaryInfo(DataTypes.STRING); new MockUp<CarbonDictionaryReaderImpl>() { http://git-wip-us.apache.org/repos/asf/carbondata/blob/29dc3028/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ForwardDictionaryCacheTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ForwardDictionaryCacheTest.java b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ForwardDictionaryCacheTest.java index c0b822b..89fbf2d 100644 --- a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ForwardDictionaryCacheTest.java +++ b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ForwardDictionaryCacheTest.java @@ -219,8 +219,7 @@ public class ForwardDictionaryCacheTest extends AbstractDictionaryCacheTest { ColumnIdentifier columnIdentifier = new ColumnIdentifier(columnId, null, null); DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier = new DictionaryColumnUniqueIdentifier(absoluteTableIdentifier, columnIdentifier, - columnIdentifier.getDataType(), - CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier)); + columnIdentifier.getDataType()); Map<String, Integer> dataToSurrogateKeyMap = new HashMap<>(data.size()); int surrogateKey = 0; List<Integer> invertedIndexList = new ArrayList<>(data.size()); http://git-wip-us.apache.org/repos/asf/carbondata/blob/29dc3028/core/src/test/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImplTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImplTest.java b/core/src/test/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImplTest.java index 7450382..70f1740 100644 --- a/core/src/test/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImplTest.java +++ b/core/src/test/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImplTest.java @@ -53,8 +53,7 @@ public class CarbonDictionaryReaderImplTest { new CarbonTableIdentifier("dbName", "tableName", UUID.randomUUID().toString())); DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier = new DictionaryColumnUniqueIdentifier(absoluteTableIdentifier, columnIdentifier, - columnIdentifier.getDataType(), - CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier)); + columnIdentifier.getDataType()); carbonDictionaryReaderImpl = new CarbonDictionaryReaderImpl(dictionaryColumnUniqueIdentifier); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/29dc3028/core/src/test/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImplTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImplTest.java b/core/src/test/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImplTest.java index b4c791c..68d34cf 100644 --- a/core/src/test/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImplTest.java +++ b/core/src/test/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImplTest.java @@ -70,8 +70,7 @@ public class CarbonDictionarySortIndexReaderImplTest { ColumnIdentifier columnIdentifier = new ColumnIdentifier("Name", null, null); DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier = new DictionaryColumnUniqueIdentifier(absoluteTableIdentifier, columnIdentifier, - columnIdentifier.getDataType(), - CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier)); + columnIdentifier.getDataType()); CarbonDictionaryWriter dictionaryWriter = new CarbonDictionaryWriterImpl(dictionaryColumnUniqueIdentifier); String metaFolderPath = http://git-wip-us.apache.org/repos/asf/carbondata/blob/29dc3028/core/src/test/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImplTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImplTest.java b/core/src/test/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImplTest.java index 4fca00e..308d041 100644 --- a/core/src/test/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImplTest.java +++ b/core/src/test/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImplTest.java @@ -106,8 +106,7 @@ public class CarbonDictionaryWriterImplTest { absoluteTableIdentifier = AbsoluteTableIdentifier.from(carbonStorePath, carbonTableIdentifier); this.dictionaryColumnUniqueIdentifier = new DictionaryColumnUniqueIdentifier(absoluteTableIdentifier, columnIdentifier, - columnIdentifier.getDataType(), - CarbonStorePath.getCarbonTablePath(carbonStorePath, carbonTableIdentifier)); + columnIdentifier.getDataType()); deleteStorePath(); prepareDataSet(); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/29dc3028/core/src/test/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImplTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImplTest.java b/core/src/test/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImplTest.java index 2b5cc85..f74a254 100644 --- a/core/src/test/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImplTest.java +++ b/core/src/test/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImplTest.java @@ -63,8 +63,7 @@ public class CarbonDictionarySortIndexWriterImplTest { columnIdentifier = new ColumnIdentifier("Name", null, null); DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier = new DictionaryColumnUniqueIdentifier(absoluteTableIdentifier, columnIdentifier, - columnIdentifier.getDataType(), - CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier)); + columnIdentifier.getDataType()); dictionaryWriter = new CarbonDictionaryWriterImpl(dictionaryColumnUniqueIdentifier); dictionarySortIndexWriter = http://git-wip-us.apache.org/repos/asf/carbondata/blob/29dc3028/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java ---------------------------------------------------------------------- diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java index bf25426..1191a38 100644 --- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java +++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java @@ -79,8 +79,7 @@ public class CarbonRecordReader<T> extends AbstractRecordReader<T> { } List<TableBlockInfo> tableBlockInfoList = CarbonInputSplit.createBlocks(splitList); queryModel.setTableBlockInfos(tableBlockInfoList); - readSupport.initialize(queryModel.getProjectionColumns(), - queryModel.getAbsoluteTableIdentifier()); + readSupport.initialize(queryModel.getProjectionColumns(), queryModel.getTable()); try { carbonIterator = new ChunkRowIterator(queryExecutor.execute(queryModel)); } catch (QueryExecutionException e) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/29dc3028/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/CarbonReadSupport.java ---------------------------------------------------------------------- diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/CarbonReadSupport.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/CarbonReadSupport.java index b535aea..c126e95 100644 --- a/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/CarbonReadSupport.java +++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/CarbonReadSupport.java @@ -18,7 +18,7 @@ package org.apache.carbondata.hadoop.readsupport; import java.io.IOException; -import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; +import org.apache.carbondata.core.metadata.schema.table.CarbonTable; import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn; /** @@ -30,10 +30,9 @@ public interface CarbonReadSupport<T> { * Initialization if needed based on the projected column list * * @param carbonColumns column list - * @param absoluteTableIdentifier table identifier + * @param carbonTable table identifier */ - void initialize(CarbonColumn[] carbonColumns, - AbsoluteTableIdentifier absoluteTableIdentifier) throws IOException; + void initialize(CarbonColumn[] carbonColumns, CarbonTable carbonTable) throws IOException; /** * convert column data back to row representation http://git-wip-us.apache.org/repos/asf/carbondata/blob/29dc3028/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/DictionaryDecodeReadSupport.java ---------------------------------------------------------------------- diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/DictionaryDecodeReadSupport.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/DictionaryDecodeReadSupport.java index f4927dd..9851fe8 100644 --- a/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/DictionaryDecodeReadSupport.java +++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/DictionaryDecodeReadSupport.java @@ -23,12 +23,12 @@ import org.apache.carbondata.core.cache.CacheProvider; import org.apache.carbondata.core.cache.CacheType; import org.apache.carbondata.core.cache.dictionary.Dictionary; import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier; -import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; +import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.metadata.datatype.DataType; import org.apache.carbondata.core.metadata.encoder.Encoding; +import org.apache.carbondata.core.metadata.schema.table.CarbonTable; import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn; import org.apache.carbondata.core.util.CarbonUtil; -import org.apache.carbondata.core.util.path.CarbonStorePath; import org.apache.carbondata.hadoop.readsupport.CarbonReadSupport; /** @@ -49,10 +49,10 @@ public class DictionaryDecodeReadSupport<T> implements CarbonReadSupport<T> { * for column dictionary involved in decoding. * * @param carbonColumns column list - * @param absoluteTableIdentifier table identifier + * @param carbonTable table identifier */ @Override public void initialize(CarbonColumn[] carbonColumns, - AbsoluteTableIdentifier absoluteTableIdentifier) throws IOException { + CarbonTable carbonTable) throws IOException { this.carbonColumns = carbonColumns; dictionaries = new Dictionary[carbonColumns.length]; dataTypes = new DataType[carbonColumns.length]; @@ -63,10 +63,11 @@ public class DictionaryDecodeReadSupport<T> implements CarbonReadSupport<T> { Cache<DictionaryColumnUniqueIdentifier, Dictionary> forwardDictionaryCache = cacheProvider .createCache(CacheType.FORWARD_DICTIONARY); dataTypes[i] = carbonColumns[i].getDataType(); + String dictionaryPath = carbonTable.getTableInfo().getFactTable().getTableProperties() + .get(CarbonCommonConstants.DICTIONARY_PATH); dictionaries[i] = forwardDictionaryCache.get(new DictionaryColumnUniqueIdentifier( - absoluteTableIdentifier, - carbonColumns[i].getColumnIdentifier(), dataTypes[i], - CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier))); + carbonTable.getAbsoluteTableIdentifier(), + carbonColumns[i].getColumnIdentifier(), dataTypes[i], dictionaryPath)); } else { dataTypes[i] = carbonColumns[i].getDataType(); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/29dc3028/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/RawDataReadSupport.java ---------------------------------------------------------------------- diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/RawDataReadSupport.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/RawDataReadSupport.java index 7e7d414..b2cd450 100644 --- a/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/RawDataReadSupport.java +++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/RawDataReadSupport.java @@ -16,7 +16,7 @@ */ package org.apache.carbondata.hadoop.readsupport.impl; -import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; +import org.apache.carbondata.core.metadata.schema.table.CarbonTable; import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn; import org.apache.carbondata.hadoop.readsupport.CarbonReadSupport; @@ -26,8 +26,7 @@ import org.apache.spark.sql.catalyst.expressions.GenericInternalRow; public class RawDataReadSupport implements CarbonReadSupport<InternalRow> { @Override - public void initialize(CarbonColumn[] carbonColumns, - AbsoluteTableIdentifier absoluteTableIdentifier) { } + public void initialize(CarbonColumn[] carbonColumns, CarbonTable carbonTable) { } /** * return column data as InternalRow http://git-wip-us.apache.org/repos/asf/carbondata/blob/29dc3028/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamInputFormat.java ---------------------------------------------------------------------- diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamInputFormat.java index e6feb93..a6e9563 100644 --- a/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamInputFormat.java +++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamInputFormat.java @@ -22,6 +22,7 @@ import java.io.IOException; import org.apache.carbondata.core.cache.Cache; import org.apache.carbondata.core.cache.dictionary.Dictionary; import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier; +import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.metadata.datatype.DataType; import org.apache.carbondata.core.metadata.datatype.DataTypes; import org.apache.carbondata.core.metadata.encoder.Encoding; @@ -33,7 +34,6 @@ import org.apache.carbondata.core.scan.complextypes.PrimitiveQueryType; import org.apache.carbondata.core.scan.complextypes.StructQueryType; import org.apache.carbondata.core.scan.filter.GenericQueryType; import org.apache.carbondata.core.util.CarbonUtil; -import org.apache.carbondata.core.util.path.CarbonStorePath; import org.apache.hadoop.mapreduce.InputSplit; import org.apache.hadoop.mapreduce.RecordReader; @@ -95,10 +95,11 @@ public class CarbonStreamInputFormat extends FileInputFormat<Void, Object> { } else { boolean isDirectDictionary = CarbonUtil.hasEncoding(child.getEncoder(), Encoding.DIRECT_DICTIONARY); + String dictionaryPath = carbontable.getTableInfo().getFactTable().getTableProperties() + .get(CarbonCommonConstants.DICTIONARY_PATH); DictionaryColumnUniqueIdentifier dictionarIdentifier = new DictionaryColumnUniqueIdentifier(carbontable.getAbsoluteTableIdentifier(), - child.getColumnIdentifier(), child.getDataType(), - CarbonStorePath.getCarbonTablePath(carbontable.getAbsoluteTableIdentifier())); + child.getColumnIdentifier(), child.getDataType(), dictionaryPath); queryType = new PrimitiveQueryType(child.getColName(), dimension.getColName(), ++parentBlockIndex, http://git-wip-us.apache.org/repos/asf/carbondata/blob/29dc3028/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java ---------------------------------------------------------------------- diff --git a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java index bea1d5e..ab22945 100644 --- a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java +++ b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java @@ -334,9 +334,7 @@ public class StoreCreator { ColumnIdentifier columnIdentifier = new ColumnIdentifier(dims.get(i).getColumnId(), null, null); DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier = new DictionaryColumnUniqueIdentifier(table.getAbsoluteTableIdentifier(), columnIdentifier, - columnIdentifier.getDataType(), CarbonStorePath - .getCarbonTablePath(table.getAbsoluteTableIdentifier().getTablePath(), - table.getCarbonTableIdentifier())); + columnIdentifier.getDataType()); CarbonDictionaryWriter writer = new CarbonDictionaryWriterImpl(dictionaryColumnUniqueIdentifier); for (String value : set[i]) { @@ -346,9 +344,7 @@ public class StoreCreator { writer.commit(); Dictionary dict = (Dictionary) dictCache.get( new DictionaryColumnUniqueIdentifier(absoluteTableIdentifier, - columnIdentifier, dims.get(i).getDataType(),CarbonStorePath - .getCarbonTablePath(table.getAbsoluteTableIdentifier().getTablePath(), - table.getCarbonTableIdentifier()))); + columnIdentifier, dims.get(i).getDataType())); CarbonDictionarySortInfoPreparator preparator = new CarbonDictionarySortInfoPreparator(); List<String> newDistinctValues = new ArrayList<String>(); http://git-wip-us.apache.org/repos/asf/carbondata/blob/29dc3028/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java ---------------------------------------------------------------------- diff --git a/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java b/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java index 2f770cd..febca50 100644 --- a/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java +++ b/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java @@ -28,10 +28,11 @@ import org.apache.carbondata.core.cache.CacheProvider; import org.apache.carbondata.core.cache.CacheType; import org.apache.carbondata.core.cache.dictionary.Dictionary; import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier; -import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; +import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.metadata.datatype.DataType; import org.apache.carbondata.core.metadata.datatype.DataTypes; import org.apache.carbondata.core.metadata.encoder.Encoding; +import org.apache.carbondata.core.metadata.schema.table.CarbonTable; import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn; import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension; import org.apache.carbondata.core.util.CarbonUtil; @@ -71,10 +72,10 @@ public class CarbonDictionaryDecodeReadSupport<T> implements CarbonReadSupport<T * for column dictionary involved in decoding. * * @param carbonColumns column list - * @param absoluteTableIdentifier table identifier + * @param carbonTable table identifier */ @Override public void initialize(CarbonColumn[] carbonColumns, - AbsoluteTableIdentifier absoluteTableIdentifier) throws IOException { + CarbonTable carbonTable) throws IOException { this.carbonColumns = carbonColumns; dictionaries = new Dictionary[carbonColumns.length]; dataTypes = new DataType[carbonColumns.length]; @@ -85,9 +86,11 @@ public class CarbonDictionaryDecodeReadSupport<T> implements CarbonReadSupport<T Cache<DictionaryColumnUniqueIdentifier, Dictionary> forwardDictionaryCache = cacheProvider .createCache(CacheType.FORWARD_DICTIONARY); dataTypes[i] = carbonColumns[i].getDataType(); + String dictionaryPath = carbonTable.getTableInfo().getFactTable().getTableProperties() + .get(CarbonCommonConstants.DICTIONARY_PATH); dictionaries[i] = forwardDictionaryCache.get( - new DictionaryColumnUniqueIdentifier(absoluteTableIdentifier, - carbonColumns[i].getColumnIdentifier())); + new DictionaryColumnUniqueIdentifier(carbonTable.getAbsoluteTableIdentifier(), + carbonColumns[i].getColumnIdentifier(), dataTypes[i], dictionaryPath)); } else { dataTypes[i] = carbonColumns[i].getDataType(); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/29dc3028/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonHiveRecordReader.java ---------------------------------------------------------------------- diff --git a/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonHiveRecordReader.java b/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonHiveRecordReader.java index b8bd0e2..57bcca3 100644 --- a/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonHiveRecordReader.java +++ b/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonHiveRecordReader.java @@ -80,7 +80,7 @@ class CarbonHiveRecordReader extends CarbonRecordReader<ArrayWritable> List<TableBlockInfo> tableBlockInfoList = CarbonHiveInputSplit.createBlocks(splitList); queryModel.setTableBlockInfos(tableBlockInfoList); readSupport - .initialize(queryModel.getProjectionColumns(), queryModel.getAbsoluteTableIdentifier()); + .initialize(queryModel.getProjectionColumns(), queryModel.getTable()); try { carbonIterator = new ChunkRowIterator(queryExecutor.execute(queryModel)); } catch (QueryExecutionException e) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/29dc3028/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSet.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSet.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSet.java index a9e2094..fb2e06e 100755 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSet.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSet.java @@ -76,7 +76,7 @@ public class CarbondataRecordSet implements RecordSet { try { readSupport - .initialize(queryModel.getProjectionColumns(), queryModel.getAbsoluteTableIdentifier()); + .initialize(queryModel.getProjectionColumns(), queryModel.getTable()); CarbonIterator iterator = queryExecutor.execute(queryModel); CarbonVectorizedRecordReader vectorReader = new CarbonVectorizedRecordReader(queryExecutor, queryModel, http://git-wip-us.apache.org/repos/asf/carbondata/blob/29dc3028/integration/presto/src/main/scala/org/apache/carbondata/presto/CarbonDictionaryDecodeReadSupport.scala ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/scala/org/apache/carbondata/presto/CarbonDictionaryDecodeReadSupport.scala b/integration/presto/src/main/scala/org/apache/carbondata/presto/CarbonDictionaryDecodeReadSupport.scala index c8e74a3..8e58f9a 100644 --- a/integration/presto/src/main/scala/org/apache/carbondata/presto/CarbonDictionaryDecodeReadSupport.scala +++ b/integration/presto/src/main/scala/org/apache/carbondata/presto/CarbonDictionaryDecodeReadSupport.scala @@ -22,9 +22,10 @@ import io.airlift.slice.Slices._ import org.apache.carbondata.core.cache.{Cache, CacheProvider, CacheType} import org.apache.carbondata.core.cache.dictionary.{Dictionary, DictionaryChunksWrapper, DictionaryColumnUniqueIdentifier} -import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier +import org.apache.carbondata.core.constants.CarbonCommonConstants import org.apache.carbondata.core.metadata.datatype.{DataType, DataTypes} import org.apache.carbondata.core.metadata.encoder.Encoding +import org.apache.carbondata.core.metadata.schema.table.CarbonTable import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn import org.apache.carbondata.core.util.{CarbonUtil, DataTypeUtil} import org.apache.carbondata.hadoop.readsupport.CarbonReadSupport @@ -45,8 +46,7 @@ class CarbonDictionaryDecodeReadSupport[T] extends CarbonReadSupport[T] { * @param absoluteTableIdentifier table identifier */ - override def initialize(carbonColumns: Array[CarbonColumn], - absoluteTableIdentifier: AbsoluteTableIdentifier) { + override def initialize(carbonColumns: Array[CarbonColumn], carbonTable: CarbonTable) { dictionaries = new Array[Dictionary](carbonColumns.length) dataTypes = new Array[DataType](carbonColumns.length) @@ -61,9 +61,11 @@ class CarbonDictionaryDecodeReadSupport[T] extends CarbonReadSupport[T] { cacheProvider .createCache(CacheType.FORWARD_DICTIONARY) dataTypes(index) = carbonColumn.getDataType + val dictionaryPath: String = carbonTable.getTableInfo.getFactTable.getTableProperties + .get(CarbonCommonConstants.DICTIONARY_PATH) dictionaries(index) = forwardDictionaryCache - .get(new DictionaryColumnUniqueIdentifier(absoluteTableIdentifier, - carbonColumn.getColumnIdentifier)) + .get(new DictionaryColumnUniqueIdentifier(carbonTable.getAbsoluteTableIdentifier, + carbonColumn.getColumnIdentifier, dataTypes(index), dictionaryPath)) dictionarySliceArray(index) = createSliceArrayBlock(dictionaries(index)) } http://git-wip-us.apache.org/repos/asf/carbondata/blob/29dc3028/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala ---------------------------------------------------------------------- diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala b/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala index 2e5310d..ed89be0 100644 --- a/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala +++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala @@ -357,10 +357,7 @@ object CarbonDataStoreCreator { new DictionaryColumnUniqueIdentifier( table.getAbsoluteTableIdentifier, columnIdentifier, - columnIdentifier.getDataType, - CarbonStorePath.getCarbonTablePath(table.getAbsoluteTableIdentifier.getTablePath, - table.getCarbonTableIdentifier) - ) + columnIdentifier.getDataType) val writer: CarbonDictionaryWriter = new CarbonDictionaryWriterImpl( dictionaryColumnUniqueIdentifier) for (value <- set(i)) { @@ -373,10 +370,8 @@ object CarbonDataStoreCreator { new DictionaryColumnUniqueIdentifier( absoluteTableIdentifier, columnIdentifier, - dims.get(i).getDataType, - CarbonStorePath.getCarbonTablePath(table.getAbsoluteTableIdentifier.getTablePath, - table.getCarbonTableIdentifier) - )) + dims.get(i).getDataType) + ) .asInstanceOf[Dictionary] val preparator: CarbonDictionarySortInfoPreparator = new CarbonDictionarySortInfoPreparator() http://git-wip-us.apache.org/repos/asf/carbondata/blob/29dc3028/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala index 44fc7ad..db29532 100644 --- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala +++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala @@ -347,8 +347,7 @@ class CarbonGlobalDictionaryGenerateRDD( DictionaryColumnUniqueIdentifier( model.table, model.columnIdentifier(split.index), - model.columnIdentifier(split.index).getDataType, - CarbonStorePath.getCarbonTablePath(model.table)) + model.columnIdentifier(split.index).getDataType) val pathService: PathService = CarbonCommonFactory.getPathService val carbonTablePath: CarbonTablePath = pathService http://git-wip-us.apache.org/repos/asf/carbondata/blob/29dc3028/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala index ccbc9f5..ee2c843 100644 --- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala +++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala @@ -175,8 +175,7 @@ object GlobalDictionaryUtil { DictionaryColumnUniqueIdentifier( model.table, model.columnIdentifier(columnIndex), - model.columnIdentifier(columnIndex).getDataType, - CarbonStorePath.getCarbonTablePath(model.table)) + model.columnIdentifier(columnIndex).getDataType) val writer: CarbonDictionaryWriter = dictService .getDictionaryWriter(dictionaryColumnUniqueIdentifier) try { @@ -215,8 +214,7 @@ object GlobalDictionaryUtil { DictionaryColumnUniqueIdentifier( model.table, model.columnIdentifier(i), - model.columnIdentifier(i).getDataType, - CarbonStorePath.getCarbonTablePath(model.table)) + model.columnIdentifier(i).getDataType) val set = new HashSet[String] if (model.dictFileExists(i)) { val reader: CarbonDictionaryReader = dictService.getDictionaryReader( @@ -854,8 +852,7 @@ object GlobalDictionaryUtil { DictionaryColumnUniqueIdentifier( absoluteTableIdentifier, columnIdentifier, - columnIdentifier.getDataType, - carbonTablePath) + columnIdentifier.getDataType) val parsedValue = DataTypeUtil.normalizeColumnValueForItsDataType(defaultValue, columnSchema) val valuesBuffer = new mutable.HashSet[String] if (null != parsedValue) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/29dc3028/integration/spark2/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java b/integration/spark2/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java index a4bc636..8402766 100644 --- a/integration/spark2/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java +++ b/integration/spark2/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java @@ -18,7 +18,7 @@ package org.apache.carbondata.spark.readsupport; import java.io.IOException; -import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; +import org.apache.carbondata.core.metadata.schema.table.CarbonTable; import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn; import org.apache.carbondata.hadoop.readsupport.impl.DictionaryDecodeReadSupport; @@ -28,7 +28,7 @@ import org.apache.spark.sql.catalyst.expressions.GenericInternalRow; public class SparkRowReadSupportImpl extends DictionaryDecodeReadSupport<InternalRow> { @Override public void initialize(CarbonColumn[] carbonColumns, - AbsoluteTableIdentifier absoluteTableIdentifier) throws IOException { + CarbonTable carbonTable) throws IOException { } @Override public InternalRow readRow(Object[] data) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/29dc3028/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala index f9dfd28..21992dd 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala @@ -34,13 +34,14 @@ import org.apache.spark.sql.types._ import org.apache.carbondata.core.cache.{Cache, CacheProvider, CacheType} import org.apache.carbondata.core.cache.dictionary.{Dictionary, DictionaryColumnUniqueIdentifier} -import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, ColumnIdentifier} +import org.apache.carbondata.core.constants.CarbonCommonConstants +import org.apache.carbondata.core.metadata.ColumnIdentifier import org.apache.carbondata.core.metadata.datatype.{DataTypes => CarbonDataTypes} import org.apache.carbondata.core.metadata.encoder.Encoding +import org.apache.carbondata.core.metadata.schema.table.CarbonTable import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension import org.apache.carbondata.core.scan.executor.util.QueryUtil import org.apache.carbondata.core.util.DataTypeUtil -import org.apache.carbondata.core.util.path.CarbonStorePath import org.apache.carbondata.spark.CarbonAliasDecoderRelation import org.apache.carbondata.spark.rdd.CarbonRDDWithTableInfo @@ -69,9 +70,9 @@ case class CarbonDictionaryDecoder( override def doExecute(): RDD[InternalRow] = { attachTree(this, "execute") { - val absoluteTableIdentifiers = relations.map { relation => + val tableNameToCarbonTableMapping = relations.map { relation => val carbonTable = relation.carbonRelation.carbonRelation.metaData.carbonTable - (carbonTable.getTableName, carbonTable.getAbsoluteTableIdentifier) + (carbonTable.getTableName, carbonTable) }.toMap if (CarbonDictionaryDecoder.isRequiredToDecode(getDictionaryColumnIds)) { @@ -80,7 +81,7 @@ case class CarbonDictionaryDecoder( val cacheProvider: CacheProvider = CacheProvider.getInstance val forwardDictionaryCache: Cache[DictionaryColumnUniqueIdentifier, Dictionary] = cacheProvider.createCache(CacheType.FORWARD_DICTIONARY) - val dicts: Seq[Dictionary] = getDictionary(absoluteTableIdentifiers, + val dicts: Seq[Dictionary] = getDictionary(tableNameToCarbonTableMapping, forwardDictionaryCache) val dictIndex = dicts.zipWithIndex.filter(x => x._1 != null).map(x => x._2) // add a task completion listener to clear dictionary that is a decisive factor for @@ -122,16 +123,16 @@ case class CarbonDictionaryDecoder( override def doConsume(ctx: CodegenContext, input: Seq[ExprCode], row: ExprCode): String = { - val absoluteTableIdentifiers = relations.map { relation => + val tableNameToCarbonTableMapping = relations.map { relation => val carbonTable = relation.carbonRelation.carbonRelation.metaData.carbonTable - (carbonTable.getTableName, carbonTable.getAbsoluteTableIdentifier) + (carbonTable.getTableName, carbonTable) }.toMap if (CarbonDictionaryDecoder.isRequiredToDecode(getDictionaryColumnIds)) { val cacheProvider: CacheProvider = CacheProvider.getInstance val forwardDictionaryCache: Cache[DictionaryColumnUniqueIdentifier, Dictionary] = cacheProvider.createCache(CacheType.FORWARD_DICTIONARY) - val dicts: Seq[ForwardDictionaryWrapper] = getDictionaryWrapper(absoluteTableIdentifiers, + val dicts: Seq[ForwardDictionaryWrapper] = getDictionaryWrapper(tableNameToCarbonTableMapping, forwardDictionaryCache) val exprs = child.output.map { exp => @@ -243,15 +244,16 @@ case class CarbonDictionaryDecoder( child.asInstanceOf[CodegenSupport].produce(ctx, this) } - private def getDictionary(atiMap: Map[String, AbsoluteTableIdentifier], + private def getDictionary(atiMap: Map[String, CarbonTable], cache: Cache[DictionaryColumnUniqueIdentifier, Dictionary]) = { val dicts: Seq[Dictionary] = getDictionaryColumnIds.map { f => if (f._2 != null) { try { + val dictionaryPath = atiMap(f._1).getTableInfo.getFactTable.getTableProperties + .get(CarbonCommonConstants.DICTIONARY_PATH) cache.get(new DictionaryColumnUniqueIdentifier( - atiMap(f._1), - f._2, f._3.getDataType, - CarbonStorePath.getCarbonTablePath(atiMap(f._1)))) + atiMap(f._1).getAbsoluteTableIdentifier, + f._2, f._3.getDataType, dictionaryPath)) } catch { case _: Throwable => null } @@ -262,7 +264,7 @@ case class CarbonDictionaryDecoder( dicts } - private def getDictionaryWrapper(atiMap: Map[String, AbsoluteTableIdentifier], + private def getDictionaryWrapper(atiMap: Map[String, CarbonTable], cache: Cache[DictionaryColumnUniqueIdentifier, Dictionary]) = { val allDictIdentifiers = new ArrayBuffer[DictionaryColumnUniqueIdentifier]() val dicts: Seq[ForwardDictionaryWrapper] = getDictionaryColumnIds.map { @@ -273,19 +275,22 @@ case class CarbonDictionaryDecoder( if (null != carbonDimension.getColumnSchema.getParentColumnTableRelations && !carbonDimension .getColumnSchema.getParentColumnTableRelations.isEmpty) { - (QueryUtil.getTableIdentifierForColumn(carbonDimension, atiMap(tableName)), + (QueryUtil + .getTableIdentifierForColumn(carbonDimension, + atiMap(tableName).getAbsoluteTableIdentifier), new ColumnIdentifier(carbonDimension.getColumnSchema .getParentColumnTableRelations.get(0).getColumnId, carbonDimension.getColumnProperties, carbonDimension.getDataType)) } else { - (atiMap(tableName), columnIdentifier) + (atiMap(tableName).getAbsoluteTableIdentifier, columnIdentifier) } + val dictionaryPath = atiMap(tableName).getTableInfo.getFactTable.getTableProperties + .get(CarbonCommonConstants.DICTIONARY_PATH) val dictionaryColumnUniqueIdentifier = new DictionaryColumnUniqueIdentifier( newAbsoluteTableIdentifier, newColumnIdentifier, carbonDimension.getDataType, - CarbonStorePath - .getCarbonTablePath(newAbsoluteTableIdentifier)) + dictionaryPath) allDictIdentifiers += dictionaryColumnUniqueIdentifier new ForwardDictionaryWrapper(dictionaryColumnUniqueIdentifier) } catch { @@ -504,14 +509,14 @@ class CarbonDecoderRDD( override def internalCompute(split: Partition, context: TaskContext): Iterator[InternalRow] = { val tableInfo = getTableInfo - val absoluteTableIdentifiers = relations.map { _ => - (tableInfo.getFactTable.getTableName, tableInfo.getOrCreateAbsoluteTableIdentifier()) + val tableNameToCarbonTableMapping = relations.map { _ => + (tableInfo.getFactTable.getTableName, CarbonTable.buildFromTableInfo(tableInfo)) }.toMap val cacheProvider: CacheProvider = CacheProvider.getInstance val forwardDictionaryCache: Cache[DictionaryColumnUniqueIdentifier, Dictionary] = cacheProvider.createCache(CacheType.FORWARD_DICTIONARY) - val dicts: Seq[Dictionary] = getDictionary(absoluteTableIdentifiers, + val dicts: Seq[Dictionary] = getDictionary(tableNameToCarbonTableMapping, forwardDictionaryCache) val dictIndex = dicts.zipWithIndex.filter(x => x._1 != null).map(x => x._2) // add a task completion listener to clear dictionary that is a decisive factor for @@ -547,15 +552,16 @@ class CarbonDecoderRDD( } } - private def getDictionary(atiMap: Map[String, AbsoluteTableIdentifier], + private def getDictionary(atiMap: Map[String, CarbonTable], cache: Cache[DictionaryColumnUniqueIdentifier, Dictionary]) = { val dicts: Seq[Dictionary] = getDictionaryColumnIds.map { f => if (f._2 != null) { try { + val dictionaryPath = atiMap(f._1).getTableInfo.getFactTable.getTableProperties + .get(CarbonCommonConstants.DICTIONARY_PATH) cache.get(new DictionaryColumnUniqueIdentifier( - atiMap(f._1), - f._2, f._3.getDataType, - CarbonStorePath.getCarbonTablePath(atiMap(f._1)))) + atiMap(f._1).getAbsoluteTableIdentifier, + f._2, f._3.getDataType, dictionaryPath)) } catch { case _: Throwable => null } http://git-wip-us.apache.org/repos/asf/carbondata/blob/29dc3028/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/DictionaryTestCaseUtil.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/DictionaryTestCaseUtil.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/DictionaryTestCaseUtil.scala index 8467f8d..f9519f8 100644 --- a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/DictionaryTestCaseUtil.scala +++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/DictionaryTestCaseUtil.scala @@ -43,9 +43,7 @@ object DictionaryTestCaseUtil { val tableIdentifier = new CarbonTableIdentifier(table.getDatabaseName, table.getTableName, "uniqueid") val absoluteTableIdentifier = AbsoluteTableIdentifier.from(table.getTablePath, tableIdentifier) val columnIdentifier = new DictionaryColumnUniqueIdentifier(absoluteTableIdentifier, - dimension.getColumnIdentifier, dimension.getDataType, - CarbonStorePath.getCarbonTablePath(table.getAbsoluteTableIdentifier) - ) + dimension.getColumnIdentifier, dimension.getDataType) val dict = CarbonLoaderUtil.getDictionary(columnIdentifier) assert(dict.getSurrogateKey(value) != CarbonCommonConstants.INVALID_SURROGATE_KEY) } http://git-wip-us.apache.org/repos/asf/carbondata/blob/29dc3028/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java b/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java index f70e38e..55fa23b 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java +++ b/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java @@ -43,7 +43,6 @@ import org.apache.carbondata.core.metadata.encoder.Encoding; import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension; import org.apache.carbondata.core.util.CarbonUtil; import org.apache.carbondata.core.util.DataTypeUtil; -import org.apache.carbondata.core.util.path.CarbonStorePath; import org.apache.carbondata.processing.loading.dictionary.DictionaryServerClientDictionary; import org.apache.carbondata.processing.loading.dictionary.DirectDictionary; import org.apache.carbondata.processing.loading.dictionary.PreCreatedDictionary; @@ -130,8 +129,7 @@ public class PrimitiveDataType implements GenericDataType<Object> { this.carbonDimension = carbonDimension; DictionaryColumnUniqueIdentifier identifier = new DictionaryColumnUniqueIdentifier(absoluteTableIdentifier, - carbonDimension.getColumnIdentifier(), carbonDimension.getDataType(), - CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier)); + carbonDimension.getColumnIdentifier(), carbonDimension.getDataType()); try { if (carbonDimension.hasEncoding(Encoding.DIRECT_DICTIONARY)) { dictionaryGenerator = new DirectDictionary(DirectDictionaryKeyGeneratorFactory
