Github user KanakaKumar commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/3059#discussion_r246309331
--- Diff:
processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
---
@@ -575,19 +575,23 @@ public static Dictionary
getDictionary(AbsoluteTableIdentifier absoluteTableIden
}
// calculate the average expected size for each node
- long sizePerNode = 0;
+ long numberOfBlocksPerNode = 0;
+ if (blockInfos.size() > 0) {
+ numberOfBlocksPerNode = blockInfos.size() / numOfNodes;
+ }
+ numberOfBlocksPerNode = numberOfBlocksPerNode <= 0 ? 1 :
numberOfBlocksPerNode;
+ long dataSizePerNode = 0;
long totalFileSize = 0;
+ for (Distributable blockInfo : uniqueBlocks) {
+ totalFileSize += ((TableBlockInfo) blockInfo).getBlockLength();
+ }
+ dataSizePerNode = totalFileSize / numOfNodes;
+ long sizePerNode = 0;
if (BlockAssignmentStrategy.BLOCK_NUM_FIRST ==
blockAssignmentStrategy) {
- if (blockInfos.size() > 0) {
- sizePerNode = blockInfos.size() / numOfNodes;
- }
- sizePerNode = sizePerNode <= 0 ? 1 : sizePerNode;
+ sizePerNode = numberOfBlocksPerNode;
--- End diff --
Please don't change sizePerNode variable
---
