sv2000 commented on a change in pull request #2912: [GOBBLIN-1072] Being more
conservative on leasing containers
URL: https://github.com/apache/incubator-gobblin/pull/2912#discussion_r389197867
##########
File path:
gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnAutoScalingManager.java
##########
@@ -189,16 +224,123 @@ void runInternal() {
}
}
+ // Find all participants appearing in this cluster. Note that Helix
instances can contain cluster-manager
+ // and potentially replanner-instance.
+ Set<String> allParticipants =
getParticipants(GobblinYarnTaskRunner.class.getSimpleName());
+
+ // Find all joined participants not in-use for this round of inspection.
+ // If idle time is beyond tolerance, mark the instance as unused by
assigning timestamp as -1.
+ for (String participant : allParticipants) {
+ if (!inUseInstances.contains(participant)) {
+ instanceIdleSinceWhen.putIfAbsent(participant,
System.currentTimeMillis());
+ if (absenceUnderTolerance(participant)) {
+ inUseInstances.add(participant);
+ }
+ } else {
+ // An instance that has been previously detected as idle but now
back to in-use.
+ // Remove this instance if existed in the tracking map.
+ instanceIdleSinceWhen.remove(participant);
+ }
+ }
+
+
+
// compute the target containers as a ceiling of number of partitions
divided by the number of containers
// per partition.
int numTargetContainers = (int) Math.ceil((double)numPartitions /
this.partitionsPerContainer);
// adjust the number of target containers based on the configured min
and max container values.
numTargetContainers = Math.max(this.minContainers,
Math.min(this.maxContainers, numTargetContainers));
+ slidingFixedWindow.add(numTargetContainers);
+
log.info("There are {} containers being requested", numTargetContainers);
- this.yarnService.requestTargetNumberOfContainers(numTargetContainers,
inUseInstances);
+
this.yarnService.requestTargetNumberOfContainers(slidingFixedWindow.getMax(),
inUseInstances);
+ }
+
+ @VisibleForTesting
+ /**
+ * Pass a participant if condition hold, where the condition, by default
is that if an instance went back to
+ * active (having partition running on it) within {@link
#maxIdleTimeInMinBeforeScalingDown} mins, we will
+ * not tag that instance as "unused" and have that as the candidate for
scaling down.
+ */
+ boolean absenceUnderTolerance(String participant){
+ return System.currentTimeMillis() -
instanceIdleSinceWhen.get(participant) <
+ TimeUnit.MINUTES.toMillis(maxIdleTimeInMinBeforeScalingDown);
+ }
+ }
+
+ /**
+ * A FIFO queue with fixed size and returns maxValue among all elements
within the queue in constant time.
+ * This data structure prevent temporary fluctuation in the number of active
helix partitions as the size of queue
+ * grows and will be less sensitive when scaling down is actually required.
+ *
+ * The interface for this lass is implemented in a minimal-necessity manner
to serve only as a sliding-sized-window
+ * which captures max value. It is NOT built for general purpose.
+ */
+ static class MaxValueEvictingQueue {
+ private ArrayDeque<Integer> evictQueue;
+ private PriorityQueue<Integer> priorityQueue;
+
+ // Queue Size
+ private int maxSize;
+ private static final int DEFAULT_MAX_SIZE = 10;
+
+ // Upper-bound of value within the queue.
+ private int upperBound;
+
+ public MaxValueEvictingQueue(int maxSize, int upperBound) {
+ Preconditions.checkArgument(maxSize > 0, "maxSize has to be a value
larger than 0");
+
+ this.maxSize = maxSize;
+ this.upperBound = upperBound;
+ this.evictQueue = new ArrayDeque<>(maxSize);
+ this.priorityQueue = new PriorityQueue<>(maxSize, new
Comparator<Integer>() {
+ @Override
+ public int compare(Integer o1, Integer o2) {
+ return o2.compareTo(o1);
+ }
+ });
+ }
+
+ public MaxValueEvictingQueue(int upperBound) {
+ this(DEFAULT_MAX_SIZE, upperBound);
+ }
+
+ /**
+ * Add element into data structure.
+ * When a new element is larger than value-upper-bound, reject the value
for safety consideration.
Review comment:
Also, instead of "safety consideration", better be explicit about what
condition you are trying to protect against. E.g. "we reject the element, since
we may request too many Yarn containers".
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
