himanshug commented on a change in pull request #9350: Overlord to support
autoscalers per indexer/middlemanager category
URL: https://github.com/apache/druid/pull/9350#discussion_r391175284
##########
File path:
indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/PendingTaskBasedWorkerProvisioningStrategy.java
##########
@@ -157,24 +125,94 @@ public synchronized boolean doProvision()
Collection<ImmutableWorkerInfo> workers = runner.getWorkers();
log.debug("Workers: %d %s", workers.size(), workers);
boolean didProvision = false;
- final DefaultWorkerBehaviorConfig workerConfig =
getDefaultWorkerBehaviorConfig(workerConfigRef, "provision", log);
+ final DefaultWorkerBehaviorConfig workerConfig =
ProvisioningUtil.getDefaultWorkerBehaviorConfig(
+ workerConfigRef,
+ "provision"
+ );
if (workerConfig == null) {
+ log.info("No worker config found. Skip provisioning.");
return false;
}
+ WorkerCategorySpec workerCategorySpec =
ProvisioningUtil.getWorkerCategorySpec(workerConfig);
+
+ // Group tasks by categories
+ Map<String, List<Task>> tasksByCategories =
pendingTasks.stream().collect(Collectors.groupingBy(
+ task -> WorkerSelectUtils.getTaskCategory(
+ task,
+ workerCategorySpec,
+ DefaultWorkerBehaviorConfig.DEFAULT_AUTOSCALER_CATEGORY
+ )
+ ));
+
+ Map<String, List<ImmutableWorkerInfo>> workersByCategories =
ProvisioningUtil.getWorkersByCategories(workers);
+
+ // Merge categories of tasks and workers
+ Set<String> allCategories = new HashSet<>(tasksByCategories.keySet());
+ allCategories.addAll(workersByCategories.keySet());
+
+ log.debug(
+ "Pending Tasks of %d categories (%s), Workers of %d categories (%s).
%d common categories: %s",
+ tasksByCategories.size(),
+ tasksByCategories.keySet(),
+ workersByCategories.size(),
+ workersByCategories.keySet(),
+ allCategories.size(),
+ allCategories
+ );
+
+ if (allCategories.isEmpty()) {
+ // Likely empty categories means initialization.
+ // Just try to spinup required amount of workers of each non empty
autoscalers
+ return initAutoscalers(workerConfig);
+ }
+
+ Map<String, AutoScaler> autoscalersByCategory =
ProvisioningUtil.mapAutoscalerByCategory(workerConfig.getAutoScalers());
+
+ for (String category : allCategories) {
+ AutoScaler categoryAutoscaler =
ProvisioningUtil.getAutoscalerByCategory(category, autoscalersByCategory);
+ if (categoryAutoscaler == null) {
+ log.error("No autoScaler available, cannot execute doProvision for
workers of category %s", category);
+ continue;
+ }
+ // Correct category name by selected autoscaler
+ category = ProvisioningUtil.getAutoscalerCategory(categoryAutoscaler);
+
+ List<Task> categoryTasks = tasksByCategories.getOrDefault(category,
Collections.emptyList());
+ List<ImmutableWorkerInfo> categoryWorkers =
workersByCategories.getOrDefault(category, Collections.emptyList());
+ currentlyProvisioningMap.putIfAbsent(category, new HashSet<>());
+ Set<String> currentlyProvisioning =
this.currentlyProvisioningMap.get(category);
Review comment:
nit: following replacement avoids instantiation of a new HashSet object on
each call to this method.
```suggestion
Set<String> currentlyProvisioning =
currentlyProvisioningMap.computeIfAbsent(
category,
ignored -> new HashSet<>()
);
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
