[2/2] carbondata git commit: [CARBONDATA-2310] Refactored code to improve Distributable interface & [CARBONDATA-2362] Changing the Cacheable object from DataMap to Wrapper
[CARBONDATA-2310] Refactored code to improve Distributable interface & [CARBONDATA-2362] Changing the Cacheable object from DataMap to Wrapper This PR has two JIRA fixes [CARBONDATA-2310] Refactored code to improve Distributable interface [CARBONDATA-2362] Changing the Cacheable object from DataMap to Wrapper This closes #2244 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/531ecdf3 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/531ecdf3 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/531ecdf3 Branch: refs/heads/master Commit: 531ecdf3f40c064d4ff6ad36c43fa90a2d423588 Parents: a7926ea Author: dhatchayaniAuthored: Fri Apr 27 23:03:52 2018 +0530 Committer: ravipesala Committed: Mon May 7 13:11:29 2018 +0530 -- .../org/apache/carbondata/core/cache/Cache.java | 10 + .../dictionary/AbstractDictionaryCache.java | 6 + .../core/constants/CarbonCommonConstants.java | 3 + .../core/datamap/dev/CacheableDataMap.java | 51 .../core/datastore/SegmentTaskIndexStore.java | 7 + .../filesystem/AbstractDFSCarbonFile.java | 5 +- .../core/datastore/filesystem/CarbonFile.java | 3 +- .../datastore/filesystem/LocalCarbonFile.java | 3 +- .../core/indexstore/AbstractMemoryDMStore.java | 63 + .../indexstore/BlockletDataMapIndexStore.java | 187 --- .../indexstore/BlockletDataMapIndexWrapper.java | 52 + .../core/indexstore/BlockletDetailInfo.java | 66 -- .../core/indexstore/SafeMemoryDMStore.java | 105 + .../TableBlockIndexUniqueIdentifier.java| 5 +- .../core/indexstore/UnsafeMemoryDMStore.java| 25 +- .../blockletindex/BlockletDataMap.java | 232 --- .../BlockletDataMapDistributable.java | 12 + .../blockletindex/BlockletDataMapFactory.java | 127 ++ .../blockletindex/BlockletDataMapModel.java | 12 + .../blockletindex/SegmentIndexFileStore.java| 39 +++- .../core/indexstore/row/DataMapRow.java | 13 +- .../core/indexstore/row/UnsafeDataMapRow.java | 7 +- .../core/indexstore/schema/CarbonRowSchema.java | 10 +- .../core/metadata/SegmentFileStore.java | 29 +++ .../core/metadata/schema/table/TableInfo.java | 24 ++ .../TableStatusReadCommittedScope.java | 4 +- .../core/util/BlockletDataMapUtil.java | 180 ++ .../carbondata/core/util/SessionParams.java | 5 + .../core/util/path/CarbonTablePath.java | 2 +- .../TestBlockletDataMapFactory.java | 126 ++ .../apache/carbondata/hadoop/CacheClient.java | 49 .../hadoop/api/AbstractDataMapJob.java | 42 .../hadoop/api/CarbonFileInputFormat.java | 2 +- .../hadoop/api/CarbonInputFormat.java | 27 ++- .../hadoop/api/CarbonTableInputFormat.java | 2 +- .../carbondata/hadoop/api/DataMapJob.java | 6 + .../hadoop/util/CarbonInputFormatUtil.java | 43 +++- .../lucene/LuceneFineGrainDataMapSuite.scala| 1 + .../carbondata/spark/rdd/CarbonScanRDD.scala| 15 +- .../carbondata/spark/rdd/SparkDataMapJob.scala | 4 +- .../org/apache/spark/sql/CarbonCountStar.scala | 3 + .../execution/command/CarbonHiveCommands.scala | 9 + 42 files changed, 1335 insertions(+), 281 deletions(-) -- http://git-wip-us.apache.org/repos/asf/carbondata/blob/531ecdf3/core/src/main/java/org/apache/carbondata/core/cache/Cache.java -- diff --git a/core/src/main/java/org/apache/carbondata/core/cache/Cache.java b/core/src/main/java/org/apache/carbondata/core/cache/Cache.java index 04fa18a..6df36fc 100644 --- a/core/src/main/java/org/apache/carbondata/core/cache/Cache.java +++ b/core/src/main/java/org/apache/carbondata/core/cache/Cache.java @@ -20,6 +20,8 @@ package org.apache.carbondata.core.cache; import java.io.IOException; import java.util.List; +import org.apache.carbondata.core.memory.MemoryException; + /** * A semi-persistent mapping from keys to values. Cache entries are manually added using * #get(Key), #getAll(List) , and are stored in the cache until @@ -69,6 +71,14 @@ public interface Cache { void invalidate(K key); /** + * This method will add the value to the cache for the given key + * + * @param key + * @param value + */ + void put(K key, V value) throws IOException, MemoryException; + + /** * Access count of Cacheable entry will be decremented * * @param keys http://git-wip-us.apache.org/repos/asf/carbondata/blob/531ecdf3/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java
[2/2] carbondata git commit: [CARBONDATA-2310] Refactored code to improve Distributable interface
[CARBONDATA-2310] Refactored code to improve Distributable interface Refactored code to improve Distributable interface This closes #2134 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/3c48df39 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/3c48df39 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/3c48df39 Branch: refs/heads/branch-1.3 Commit: 3c48df396f2bafc9efc8091fc7abefca089922d7 Parents: 31c7b50 Author: dhatchayaniAuthored: Tue Apr 3 11:19:43 2018 +0530 Committer: manishgupta88 Committed: Tue Apr 10 16:16:27 2018 +0530 -- .../org/apache/carbondata/core/cache/Cache.java | 10 ++ .../dictionary/AbstractDictionaryCache.java | 4 + .../core/constants/CarbonCommonConstants.java | 3 + .../core/datamap/dev/CacheableDataMap.java | 47 ++ .../carbondata/core/datamap/dev/DataMap.java| 3 +- .../core/datastore/BlockIndexStore.java | 4 + .../core/datastore/SegmentTaskIndexStore.java | 4 + .../core/indexstore/AbstractMemoryDMStore.java | 63 +++ .../indexstore/BlockletDataMapIndexStore.java | 92 -- .../core/indexstore/SafeMemoryDMStore.java | 94 +++ .../TableBlockIndexUniqueIdentifier.java| 3 +- .../core/indexstore/UnsafeMemoryDMStore.java| 23 +-- .../blockletindex/BlockletDataMap.java | 169 +-- .../BlockletDataMapDistributable.java | 18 +- .../blockletindex/BlockletDataMapFactory.java | 95 +++ .../blockletindex/BlockletDataMapModel.java | 13 ++ .../core/indexstore/row/DataMapRow.java | 13 +- .../core/indexstore/row/UnsafeDataMapRow.java | 7 +- .../core/indexstore/schema/CarbonRowSchema.java | 4 +- .../core/util/BlockletDataMapUtil.java | 140 +++ .../carbondata/core/util/SessionParams.java | 5 + .../TestBlockletDataMapFactory.java | 108 .../apache/carbondata/hadoop/CacheClient.java | 43 + .../hadoop/api/AbstractDataMapJob.java | 43 + .../hadoop/api/CarbonTableInputFormat.java | 61 ++- .../carbondata/hadoop/api/DataMapJob.java | 6 + .../hadoop/util/CarbonInputFormatUtil.java | 44 + .../carbondata/spark/rdd/CarbonScanRDD.scala| 9 +- .../carbondata/spark/rdd/SparkDataMapJob.scala | 4 +- .../org/apache/spark/sql/CarbonCountStar.scala | 13 ++ .../execution/command/CarbonHiveCommands.scala | 9 + 31 files changed, 973 insertions(+), 181 deletions(-) -- http://git-wip-us.apache.org/repos/asf/carbondata/blob/3c48df39/core/src/main/java/org/apache/carbondata/core/cache/Cache.java -- diff --git a/core/src/main/java/org/apache/carbondata/core/cache/Cache.java b/core/src/main/java/org/apache/carbondata/core/cache/Cache.java index 04fa18a..6df36fc 100644 --- a/core/src/main/java/org/apache/carbondata/core/cache/Cache.java +++ b/core/src/main/java/org/apache/carbondata/core/cache/Cache.java @@ -20,6 +20,8 @@ package org.apache.carbondata.core.cache; import java.io.IOException; import java.util.List; +import org.apache.carbondata.core.memory.MemoryException; + /** * A semi-persistent mapping from keys to values. Cache entries are manually added using * #get(Key), #getAll(List) , and are stored in the cache until @@ -69,6 +71,14 @@ public interface Cache { void invalidate(K key); /** + * This method will add the value to the cache for the given key + * + * @param key + * @param value + */ + void put(K key, V value) throws IOException, MemoryException; + + /** * Access count of Cacheable entry will be decremented * * @param keys http://git-wip-us.apache.org/repos/asf/carbondata/blob/3c48df39/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java -- diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java index 598d00e..9ed9007 100644 --- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java +++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java @@ -59,6 +59,10 @@ public abstract class AbstractDictionaryCachehttp://git-wip-us.apache.org/repos/asf/carbondata/blob/3c48df39/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java -- diff --git