Github user jackylk commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1808#discussion_r167119617
--- Diff:
processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
---
@@ -606,122 +609,296 @@ private static void
createTaskListForNode(Map<String, List<List<Distributable>>>
}
/**
- * If any left over data blocks are present then assign those to nodes
in round robin way.
+ * If any left over data blocks are present then assign those to nodes
in round robin way. This
+ * will not obey the data locality.
*
* @param outputMap
- * @param uniqueBlocks
+ * @param leftOverBlocks
*/
- private static void assignLeftOverBlocks(Map<String,
List<Distributable>> outputMap,
- Set<Distributable> uniqueBlocks, int noOfBlocksPerNode, List<String>
activeNodes) {
+ private static void
assignLeftOverBlocks(ArrayList<NodeMultiBlockRelation> outputMap,
+ Set<Distributable> leftOverBlocks, long expectedSizePerNode,
List<String> activeNodes,
+ BlockAssignmentStrategy blockAssignmentStrategy) {
+ Map<String, Integer> node2Idx = new HashMap<>(outputMap.size());
+ for (int idx = 0; idx < outputMap.size(); idx++) {
+ node2Idx.put(outputMap.get(idx).getNode(), idx);
+ }
+ // iterate all the nodes and try to allocate blocks to the nodes
if (activeNodes != null) {
for (String activeNode : activeNodes) {
- List<Distributable> blockLst = outputMap.get(activeNode);
- if (null == blockLst) {
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("Second assignment iteration: assign for executor:
" + activeNode);
+ }
+
+ Integer idx;
+ List<Distributable> blockLst;
+ if (node2Idx.containsKey(activeNode)) {
+ idx = node2Idx.get(activeNode);
+ blockLst = outputMap.get(idx).getBlocks();
+ } else {
+ idx = node2Idx.size();
blockLst = new
ArrayList<Distributable>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
}
- populateBlocks(uniqueBlocks, noOfBlocksPerNode, blockLst);
- if (blockLst.size() > 0) {
- outputMap.put(activeNode, blockLst);
+ populateBlocks(leftOverBlocks, expectedSizePerNode, blockLst,
blockAssignmentStrategy);
+
+ if (!node2Idx.containsKey(activeNode) && blockLst.size() > 0) {
+ outputMap.add(idx, new NodeMultiBlockRelation(activeNode,
blockLst));
+ node2Idx.put(activeNode, idx);
}
}
} else {
- for (Map.Entry<String, List<Distributable>> entry :
outputMap.entrySet()) {
- List<Distributable> blockLst = entry.getValue();
- populateBlocks(uniqueBlocks, noOfBlocksPerNode, blockLst);
+ for (NodeMultiBlockRelation entry : outputMap) {
+ List<Distributable> blockLst = entry.getBlocks();
+ populateBlocks(leftOverBlocks, expectedSizePerNode, blockLst,
blockAssignmentStrategy);
}
-
}
- for (Map.Entry<String, List<Distributable>> entry :
outputMap.entrySet()) {
- Iterator<Distributable> blocks = uniqueBlocks.iterator();
- if (blocks.hasNext()) {
- Distributable block = blocks.next();
- List<Distributable> blockLst = entry.getValue();
- blockLst.add(block);
- blocks.remove();
- }
- }
+ // if there is still blocks left, allocate them in round robin manner
to each nodes
+ assignBlocksUseRoundRobin(outputMap, leftOverBlocks,
blockAssignmentStrategy);
}
/**
- * The method populate the blockLst to be allocate to a specific node.
- * @param uniqueBlocks
- * @param noOfBlocksPerNode
+ * assign blocks to nodes
+ * @param remainingBlocks
+ * @param expectedSizePerNode
+ * @param blockLst
+ * @param blockAssignmentStrategy
--- End diff --
Please remove @param if no description
---