Github user jackylk commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1808#discussion_r167121062
--- Diff:
processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
---
@@ -606,122 +609,296 @@ private static void
createTaskListForNode(Map<String, List<List<Distributable>>>
}
/**
- * If any left over data blocks are present then assign those to nodes
in round robin way.
+ * If any left over data blocks are present then assign those to nodes
in round robin way. This
+ * will not obey the data locality.
*
* @param outputMap
- * @param uniqueBlocks
+ * @param leftOverBlocks
*/
- private static void assignLeftOverBlocks(Map<String,
List<Distributable>> outputMap,
- Set<Distributable> uniqueBlocks, int noOfBlocksPerNode, List<String>
activeNodes) {
+ private static void
assignLeftOverBlocks(ArrayList<NodeMultiBlockRelation> outputMap,
+ Set<Distributable> leftOverBlocks, long expectedSizePerNode,
List<String> activeNodes,
+ BlockAssignmentStrategy blockAssignmentStrategy) {
+ Map<String, Integer> node2Idx = new HashMap<>(outputMap.size());
+ for (int idx = 0; idx < outputMap.size(); idx++) {
+ node2Idx.put(outputMap.get(idx).getNode(), idx);
+ }
+ // iterate all the nodes and try to allocate blocks to the nodes
if (activeNodes != null) {
for (String activeNode : activeNodes) {
- List<Distributable> blockLst = outputMap.get(activeNode);
- if (null == blockLst) {
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("Second assignment iteration: assign for executor:
" + activeNode);
+ }
+
+ Integer idx;
+ List<Distributable> blockLst;
+ if (node2Idx.containsKey(activeNode)) {
+ idx = node2Idx.get(activeNode);
+ blockLst = outputMap.get(idx).getBlocks();
+ } else {
+ idx = node2Idx.size();
blockLst = new
ArrayList<Distributable>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
}
- populateBlocks(uniqueBlocks, noOfBlocksPerNode, blockLst);
- if (blockLst.size() > 0) {
- outputMap.put(activeNode, blockLst);
+ populateBlocks(leftOverBlocks, expectedSizePerNode, blockLst,
blockAssignmentStrategy);
+
+ if (!node2Idx.containsKey(activeNode) && blockLst.size() > 0) {
+ outputMap.add(idx, new NodeMultiBlockRelation(activeNode,
blockLst));
+ node2Idx.put(activeNode, idx);
}
}
} else {
- for (Map.Entry<String, List<Distributable>> entry :
outputMap.entrySet()) {
- List<Distributable> blockLst = entry.getValue();
- populateBlocks(uniqueBlocks, noOfBlocksPerNode, blockLst);
+ for (NodeMultiBlockRelation entry : outputMap) {
+ List<Distributable> blockLst = entry.getBlocks();
+ populateBlocks(leftOverBlocks, expectedSizePerNode, blockLst,
blockAssignmentStrategy);
}
-
}
- for (Map.Entry<String, List<Distributable>> entry :
outputMap.entrySet()) {
- Iterator<Distributable> blocks = uniqueBlocks.iterator();
- if (blocks.hasNext()) {
- Distributable block = blocks.next();
- List<Distributable> blockLst = entry.getValue();
- blockLst.add(block);
- blocks.remove();
- }
- }
+ // if there is still blocks left, allocate them in round robin manner
to each nodes
+ assignBlocksUseRoundRobin(outputMap, leftOverBlocks,
blockAssignmentStrategy);
}
/**
- * The method populate the blockLst to be allocate to a specific node.
- * @param uniqueBlocks
- * @param noOfBlocksPerNode
+ * assign blocks to nodes
+ * @param remainingBlocks
+ * @param expectedSizePerNode
+ * @param blockLst
+ * @param blockAssignmentStrategy
+ */
+ private static void populateBlocks(Set<Distributable> remainingBlocks,
+ long expectedSizePerNode, List<Distributable> blockLst,
+ BlockAssignmentStrategy blockAssignmentStrategy) {
+ switch (blockAssignmentStrategy) {
+ case BLOCK_NUM_FIRST:
+ populateBlocksByNum(remainingBlocks, expectedSizePerNode,
blockLst);
+ break;
+ case BLOCK_SIZE_FIRST:
+ populateBlocksBySize(remainingBlocks, expectedSizePerNode,
blockLst);
+ break;
+ default:
+ throw new IllegalArgumentException(
+ "Unsupported block assignment strategy: " +
blockAssignmentStrategy);
+ }
+ }
+ /**
+ * allocate blocks by block num
+ * @param remainingBlocks
+ * @param expectedSizePerNode
* @param blockLst
*/
- private static void populateBlocks(Set<Distributable> uniqueBlocks, int
noOfBlocksPerNode,
- List<Distributable> blockLst) {
- Iterator<Distributable> blocks = uniqueBlocks.iterator();
- //if the node is already having the per block nodes then avoid assign
the extra blocks
- if (blockLst.size() == noOfBlocksPerNode) {
+ private static void populateBlocksByNum(Set<Distributable>
remainingBlocks,
+ long expectedSizePerNode, List<Distributable> blockLst) {
+ Iterator<Distributable> blocks = remainingBlocks.iterator();
+ // if the node is already having the per block nodes then avoid assign
the extra blocks
+ if (blockLst.size() == expectedSizePerNode) {
return;
}
while (blocks.hasNext()) {
Distributable block = blocks.next();
blockLst.add(block);
blocks.remove();
- if (blockLst.size() >= noOfBlocksPerNode) {
+ if (blockLst.size() >= expectedSizePerNode) {
+ break;
+ }
+ }
+ }
+
+ /**
+ * allocate blocks by block size
+ * @param remainingBlocks
+ * @param expectedSizePerNode
+ * @param blockLst
+ */
+ private static void populateBlocksBySize(Set<Distributable>
remainingBlocks,
+ long expectedSizePerNode, List<Distributable> blockLst) {
+ Iterator<Distributable> blocks = remainingBlocks.iterator();
+ //if the node is already having the avg node size then avoid assign
the extra blocks
+ long fileSize = 0;
+ for (Distributable block : blockLst) {
+ fileSize += ((TableBlockInfo) block).getBlockLength();
+ }
+ if (fileSize >= expectedSizePerNode) {
+ LOGGER.debug("Capacity is full, skip allocate blocks on this node");
+ return;
+ }
+
+ while (blocks.hasNext()) {
+ Distributable block = blocks.next();
+ long thisBlockSize = ((TableBlockInfo) block).getBlockLength();
+ if (fileSize < expectedSizePerNode) {
+ // `fileSize==0` means there are no blocks assigned to this node
before
+ if (fileSize == 0 || fileSize + thisBlockSize <=
expectedSizePerNode * 1.1D) {
+ blockLst.add(block);
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("Second Assignment iteration: "
+ + ((TableBlockInfo) block).getFilePath() + "-"
+ + ((TableBlockInfo) block).getBlockLength() +
"-->currentNode");
+ }
+ fileSize += thisBlockSize;
+ blocks.remove();
+ }
+ } else {
break;
}
}
}
+ /**
+ * allocate the blocks in round robin manner
+ * @param node2Blocks
+ * @param remainingBlocks
+ * @param blockAssignmentStrategy
+ */
+ private static void
assignBlocksUseRoundRobin(ArrayList<NodeMultiBlockRelation> node2Blocks,
+ Set<Distributable> remainingBlocks, BlockAssignmentStrategy
blockAssignmentStrategy) {
+ switch (blockAssignmentStrategy) {
+ case BLOCK_NUM_FIRST:
+ roundRobinAssignBlocksByNum(node2Blocks, remainingBlocks);
+ break;
+ case BLOCK_SIZE_FIRST:
+ roundRobinAssignBlocksBySize(node2Blocks, remainingBlocks);
+ break;
+ default:
+ throw new IllegalArgumentException("Unsupported block assignment
strategy: "
+ + blockAssignmentStrategy);
+ }
+ }
+
+ private static void
roundRobinAssignBlocksByNum(ArrayList<NodeMultiBlockRelation> outputMap,
+ Set<Distributable> remainingBlocks) {
+ for (NodeMultiBlockRelation relation: outputMap) {
+ Iterator<Distributable> blocks = remainingBlocks.iterator();
+ if (blocks.hasNext()) {
+ Distributable block = blocks.next();
+ List<Distributable> blockLst = relation.getBlocks();
+ blockLst.add(block);
+ blocks.remove();
+ }
+ }
+ }
+
+ private static void
roundRobinAssignBlocksBySize(ArrayList<NodeMultiBlockRelation> outputMap,
+ Set<Distributable> remainingBlocks) {
+ Iterator<Distributable> blocks = remainingBlocks.iterator();
+ while (blocks.hasNext()) {
+ // sort the allocated node-2-blocks in ascending order, the total
data size of first one is
+ // the smallest, so we assign this block to it.
+ Collections.sort(outputMap,
NodeMultiBlockRelation.DATA_SIZE_ASC_COMPARATOR);
+ Distributable block = blocks.next();
+ List<Distributable> blockLst = outputMap.get(0).getBlocks();
+ blockLst.add(block);
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("RoundRobin assignment iteration: "
+ + ((TableBlockInfo) block).getFilePath() + "-"
+ + ((TableBlockInfo) block).getBlockLength() + "-->" +
outputMap.get(0).getNode());
+ }
+ blocks.remove();
+ }
+ }
/**
* To create the final output of the Node and Data blocks
*
- * @param outputMap
- * @param blocksPerNode
- * @param uniqueBlocks
- * @param nodeAndBlockMapping
+ * @param outputNode2Blocks
+ * @param expectedSizePerNode
+ * @param remainingBlocks
+ * @param inputNode2Blocks
--- End diff --
move @param if no description
---