[gobblin] branch master updated: make the requestedContainerCountMap correctly update the container count (#3523)

[zihanli58]

This is an automated email from the ASF dual-hosted git repository.

zihanli58 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/gobblin.git


The following commit(s) were added to refs/heads/master by this push:
     new 2414bc6ac make the requestedContainerCountMap correctly update the 
container count (#3523)
2414bc6ac is described below

commit 2414bc6ac20acdf749ef2de71e2c9ba9fc90b687
Author: Hanghang Nate Liu <nate.hanghang....@gmail.com>
AuthorDate: Fri Jun 17 11:28:59 2022 -0700

    make the requestedContainerCountMap correctly update the container count 
(#3523)
    
    update the place to decrease the requestedContainerCountMap
---
 .../src/main/java/org/apache/gobblin/yarn/YarnService.java    | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git 
a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnService.java 
b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnService.java
index d265d0296..a81960fdf 100644
--- a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnService.java
+++ b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnService.java
@@ -448,14 +448,7 @@ public class YarnService extends AbstractIdleService {
       for(; requestedContainerCount < desiredContainerCount; 
requestedContainerCount++) {
         requestContainer(Optional.absent(), 
yarnContainerRequestBundle.getHelixTagResourceMap().get(currentHelixTag));
       }
-      requestedContainerCountMap.put(currentHelixTag, desiredContainerCount);
-    }
-
-    // If a requested tag is not presented in the new request, update the 
requested count to 0 as we should release them
-    for(String requestedHelixTag : requestedContainerCountMap.keySet()) {
-      
if(!yarnContainerRequestBundle.getHelixTagContainerCountMap().containsKey(requestedHelixTag))
 {
-        requestedContainerCountMap.put(requestedHelixTag, 0);
-      }
+      requestedContainerCountMap.put(currentHelixTag, requestedContainerCount);
     }
 
     // If the total desired is lower than the currently allocated amount then 
release free containers.
@@ -473,6 +466,8 @@ public class YarnService extends AbstractIdleService {
         ContainerInfo containerInfo = entry.getValue();
         if (!inUseInstances.contains(containerInfo.getHelixParticipantId())) {
           containersToRelease.add(containerInfo.getContainer());
+          requestedContainerCountMap.put(containerInfo.getHelixTag(),
+              requestedContainerCountMap.get(containerInfo.getHelixTag()) - 1);
         }
 
         if (containersToRelease.size() == numToShutdown) {