sascha-coenen commented on a change in pull request #9350: Overlord to support
autoscalers per indexer/middlemanager category
URL: https://github.com/apache/druid/pull/9350#discussion_r393606761
##########
File path:
indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/SimpleWorkerProvisioningStrategy.java
##########
@@ -184,54 +285,107 @@ public String apply(ImmutableWorkerInfo input)
@Override
public synchronized boolean doTerminate()
{
+ Collection<ImmutableWorkerInfo> workers = runner.getWorkers();
Collection<? extends TaskRunnerWorkItem> pendingTasks =
runner.getPendingTasks();
- final DefaultWorkerBehaviorConfig workerConfig =
-
PendingTaskBasedWorkerProvisioningStrategy.getDefaultWorkerBehaviorConfig(workerConfigRef,
"terminate", log);
+ final DefaultWorkerBehaviorConfig workerConfig =
ProvisioningUtil.getDefaultWorkerBehaviorConfig(
+ workerConfigRef,
+ "terminate"
+ );
if (workerConfig == null) {
+ log.info("No worker config found. Skip terminating.");
return false;
}
boolean didTerminate = false;
+
+ WorkerCategorySpec workerCategorySpec =
ProvisioningUtil.getWorkerCategorySpec(workerConfig);
+
+ // Group tasks by categories
+ Map<String, List<TaskRunnerWorkItem>> pendingTasksByCategories =
groupTasksByCategories(
+ pendingTasks,
+ runner,
+ workerCategorySpec
+ );
+
+ Map<String, List<ImmutableWorkerInfo>> workersByCategories =
ProvisioningUtil.getWorkersByCategories(workers);
+
+ Set<String> allCategories = workersByCategories.keySet();
+ log.debug(
+ "Workers of %d categories: %s",
+ workersByCategories.size(),
+ allCategories
+ );
+
+ Map<String, AutoScaler> autoscalersByCategory =
ProvisioningUtil.mapAutoscalerByCategory(workerConfig.getAutoScalers());
+
+ for (String category : allCategories) {
+ AutoScaler categoryAutoscaler =
ProvisioningUtil.getAutoscalerByCategory(category, autoscalersByCategory);
+
+ if (categoryAutoscaler == null) {
+ log.error("No autoScaler available, cannot execute doTerminate for
workers of category %s", category);
+ continue;
+ }
+
+ // Correct category name by selected autoscaler
+ category = ProvisioningUtil.getAutoscalerCategory(categoryAutoscaler);
+ List<ImmutableWorkerInfo> categoryWorkers =
workersByCategories.getOrDefault(category, Collections.emptyList());
+ currentlyProvisioningMap.putIfAbsent(category, new HashSet<>());
+ Set<String> currentlyProvisioning =
this.currentlyProvisioningMap.get(category);
+ currentlyTerminatingMap.putIfAbsent(category, new HashSet<>());
+ Set<String> currentlyTerminating =
this.currentlyTerminatingMap.get(category);
Review comment:
done
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
