[CARBONDATA-2034]Fixed Query performance issue Problem: Dictionary loading is taking more time in executor side when number of nodes is high.
Solution: During query no need to load non complex dimension dictionary. Dictionary decoder will take care of loading and decoding the dictionary column This closes #1811 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/4f9aeaea Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/4f9aeaea Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/4f9aeaea Branch: refs/heads/carbonstore Commit: 4f9aeaea8290764dc108e4e3c2df0c479f311659 Parents: 2f3f3b5 Author: kumarvishal <[email protected]> Authored: Tue Jan 16 14:37:25 2018 +0530 Committer: ravipesala <[email protected]> Committed: Wed Jan 17 18:22:44 2018 +0530 ---------------------------------------------------------------------- .../core/scan/executor/util/QueryUtil.java | 29 ++++++++++---------- 1 file changed, 15 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/4f9aeaea/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java index b33ab8d..8eb0a0d 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java @@ -281,10 +281,8 @@ public class QueryUtil { List<QueryDimension> queryDimensions, Set<CarbonDimension> filterComplexDimensions, AbsoluteTableIdentifier absoluteTableIdentifier, TableProvider tableProvider) throws IOException { - // to store dimension unique column id list, this is required as - // dimension can be present in - // query dimension, as well as some aggregation function will be applied - // in the same dimension + // to store complex dimension and its child id unique column id list, this is required as + // dimension can be present in projection and filter // so we need to get only one instance of dictionary // direct dictionary skip is done only for the dictionary lookup Set<String> dictionaryDimensionFromQuery = new HashSet<String>(); @@ -294,20 +292,19 @@ public class QueryUtil { // write encoding dictionary if (CarbonUtil.hasEncoding(encodingList, Encoding.DICTIONARY) && !CarbonUtil .hasEncoding(encodingList, Encoding.DIRECT_DICTIONARY) && !CarbonUtil - .hasEncoding(encodingList, Encoding.IMPLICIT)) { - - if (queryDimensions.get(i).getDimension().getNumberOfChild() == 0) { - dictionaryDimensionFromQuery.add(queryDimensions.get(i).getDimension().getColumnId()); - } - if (queryDimensions.get(i).getDimension().getNumberOfChild() > 0) { - getChildDimensionDictionaryDetail(queryDimensions.get(i).getDimension(), - dictionaryDimensionFromQuery); - } + .hasEncoding(encodingList, Encoding.IMPLICIT) + && queryDimensions.get(i).getDimension().getNumberOfChild() > 0) { + getChildDimensionDictionaryDetail(queryDimensions.get(i).getDimension(), + dictionaryDimensionFromQuery); } } Iterator<CarbonDimension> iterator = filterComplexDimensions.iterator(); while (iterator.hasNext()) { - getChildDimensionDictionaryDetail(iterator.next(), dictionaryDimensionFromQuery); + CarbonDimension filterDim = iterator.next(); + // only to add complex dimension + if (filterDim.getNumberOfChild() > 0) { + getChildDimensionDictionaryDetail(filterDim, dictionaryDimensionFromQuery); + } } // converting to list as api exposed needed list which i think // is not correct @@ -348,6 +345,10 @@ public class QueryUtil { private static Map<String, Dictionary> getDictionaryMap(List<String> dictionaryColumnIdList, AbsoluteTableIdentifier absoluteTableIdentifier, TableProvider tableProvider) throws IOException { + // if any complex dimension not present in query then return the empty map + if (dictionaryColumnIdList.size() == 0) { + return new HashMap<>(); + } // this for dictionary unique identifier List<DictionaryColumnUniqueIdentifier> dictionaryColumnUniqueIdentifiers = getDictionaryColumnUniqueIdentifierList(dictionaryColumnIdList,
