Github user jackylk commented on a diff in the pull request:

    https://github.com/apache/carbondata/pull/1808#discussion_r167120778
  
    --- Diff: 
processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
 ---
    @@ -606,122 +609,296 @@ private static void 
createTaskListForNode(Map<String, List<List<Distributable>>>
       }
     
       /**
    -   * If any left over data blocks are present then assign those to nodes 
in round robin way.
    +   * If any left over data blocks are present then assign those to nodes 
in round robin way. This
    +   * will not obey the data locality.
        *
        * @param outputMap
    -   * @param uniqueBlocks
    +   * @param leftOverBlocks
        */
    -  private static void assignLeftOverBlocks(Map<String, 
List<Distributable>> outputMap,
    -      Set<Distributable> uniqueBlocks, int noOfBlocksPerNode, List<String> 
activeNodes) {
    +  private static void 
assignLeftOverBlocks(ArrayList<NodeMultiBlockRelation> outputMap,
    +      Set<Distributable> leftOverBlocks, long expectedSizePerNode, 
List<String> activeNodes,
    +      BlockAssignmentStrategy blockAssignmentStrategy) {
    +    Map<String, Integer> node2Idx = new HashMap<>(outputMap.size());
    +    for (int idx = 0; idx < outputMap.size(); idx++) {
    +      node2Idx.put(outputMap.get(idx).getNode(), idx);
    +    }
     
    +    // iterate all the nodes and try to allocate blocks to the nodes
         if (activeNodes != null) {
           for (String activeNode : activeNodes) {
    -        List<Distributable> blockLst = outputMap.get(activeNode);
    -        if (null == blockLst) {
    +        if (LOGGER.isDebugEnabled()) {
    +          LOGGER.debug("Second assignment iteration: assign for executor: 
" + activeNode);
    +        }
    +
    +        Integer idx;
    +        List<Distributable> blockLst;
    +        if (node2Idx.containsKey(activeNode)) {
    +          idx = node2Idx.get(activeNode);
    +          blockLst = outputMap.get(idx).getBlocks();
    +        } else {
    +          idx = node2Idx.size();
               blockLst = new 
ArrayList<Distributable>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
             }
    -        populateBlocks(uniqueBlocks, noOfBlocksPerNode, blockLst);
    -        if (blockLst.size() > 0) {
    -          outputMap.put(activeNode, blockLst);
    +        populateBlocks(leftOverBlocks, expectedSizePerNode, blockLst, 
blockAssignmentStrategy);
    +
    +        if (!node2Idx.containsKey(activeNode) && blockLst.size() > 0) {
    +          outputMap.add(idx, new NodeMultiBlockRelation(activeNode, 
blockLst));
    +          node2Idx.put(activeNode, idx);
             }
           }
         } else {
    -      for (Map.Entry<String, List<Distributable>> entry : 
outputMap.entrySet()) {
    -        List<Distributable> blockLst = entry.getValue();
    -        populateBlocks(uniqueBlocks, noOfBlocksPerNode, blockLst);
    +      for (NodeMultiBlockRelation entry : outputMap) {
    +        List<Distributable> blockLst = entry.getBlocks();
    +        populateBlocks(leftOverBlocks, expectedSizePerNode, blockLst, 
blockAssignmentStrategy);
           }
    -
         }
     
    -    for (Map.Entry<String, List<Distributable>> entry : 
outputMap.entrySet()) {
    -      Iterator<Distributable> blocks = uniqueBlocks.iterator();
    -      if (blocks.hasNext()) {
    -        Distributable block = blocks.next();
    -        List<Distributable> blockLst = entry.getValue();
    -        blockLst.add(block);
    -        blocks.remove();
    -      }
    -    }
    +    // if there is still blocks left, allocate them in round robin manner 
to each nodes
    +    assignBlocksUseRoundRobin(outputMap, leftOverBlocks, 
blockAssignmentStrategy);
       }
     
       /**
    -   * The method populate the blockLst to be allocate to a specific node.
    -   * @param uniqueBlocks
    -   * @param noOfBlocksPerNode
    +   * assign blocks to nodes
    +   * @param remainingBlocks
    +   * @param expectedSizePerNode
    +   * @param blockLst
    +   * @param blockAssignmentStrategy
    +   */
    +  private static void populateBlocks(Set<Distributable> remainingBlocks,
    +      long expectedSizePerNode, List<Distributable> blockLst,
    +      BlockAssignmentStrategy blockAssignmentStrategy) {
    +    switch (blockAssignmentStrategy) {
    +      case BLOCK_NUM_FIRST:
    +        populateBlocksByNum(remainingBlocks, expectedSizePerNode, 
blockLst);
    +        break;
    +      case BLOCK_SIZE_FIRST:
    +        populateBlocksBySize(remainingBlocks, expectedSizePerNode, 
blockLst);
    +        break;
    +      default:
    +        throw new IllegalArgumentException(
    +            "Unsupported block assignment strategy: " + 
blockAssignmentStrategy);
    +    }
    +  }
    +  /**
    +   * allocate blocks by block num
    +   * @param remainingBlocks
    +   * @param expectedSizePerNode
        * @param blockLst
        */
    -  private static void populateBlocks(Set<Distributable> uniqueBlocks, int 
noOfBlocksPerNode,
    -      List<Distributable> blockLst) {
    -    Iterator<Distributable> blocks = uniqueBlocks.iterator();
    -    //if the node is already having the per block nodes then avoid assign 
the extra blocks
    -    if (blockLst.size() == noOfBlocksPerNode) {
    +  private static void populateBlocksByNum(Set<Distributable> 
remainingBlocks,
    +      long expectedSizePerNode, List<Distributable> blockLst) {
    +    Iterator<Distributable> blocks = remainingBlocks.iterator();
    +    // if the node is already having the per block nodes then avoid assign 
the extra blocks
    +    if (blockLst.size() == expectedSizePerNode) {
           return;
         }
         while (blocks.hasNext()) {
           Distributable block = blocks.next();
           blockLst.add(block);
           blocks.remove();
    -      if (blockLst.size() >= noOfBlocksPerNode) {
    +      if (blockLst.size() >= expectedSizePerNode) {
    +        break;
    +      }
    +    }
    +  }
    +
    +  /**
    +   * allocate blocks by block size
    --- End diff --
    
    Change to:
    Take N number of distributable blocks from remainingBlocks and add them to 
output parameter blockList. The number of blocks added is determined by 
accumulated block size is less than expectedSizePerNode



---

Reply via email to