jerrypeng commented on a change in pull request #12178:
URL: https://github.com/apache/pulsar/pull/12178#discussion_r719744518
##########
File path:
pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/SchedulerManager.java
##########
@@ -403,6 +548,82 @@ private void scheduleCompaction(ScheduledExecutorService
executor, long schedule
}
}
+ @VisibleForTesting
+ List<Assignment> getCurrentPostDrainAssignments() {
+ return currentPostDrainAssignments;
+ }
+
+ @VisibleForTesting
+ void clearCurrentPostDrainAssignments() {
+ currentPostDrainAssignments = null;
+ }
+
+ @VisibleForTesting
+ List<Assignment> invokeDrain(String workerId) {
+
+ long startTime = System.nanoTime();
+
+ Set<String> availableWorkers = getCurrentAvailableWorkers();
+
+ // workerIdToAssignments is a map of the form {workerId :
{FullyQualifiedInstanceId : Assignment}}
+ Map<String, Map<String, Assignment>> workerIdToAssignments =
functionRuntimeManager.getCurrentAssignments();
+
+ // initialize stats collection
+ SchedulerStats schedulerStats = new
SchedulerStats(workerIdToAssignments, availableWorkers);
+
+ boolean drainSuccessful = false;
+ List<Assignment> postDrainAssignments = null;
+
+ try {
+ // The worker was checked to be among the available ones, by the
caller.
+ Preconditions.checkState(availableWorkers.contains(workerId));
+
+ drainOpStatusMap.put(workerId, DrainOpStatus.DrainInProgress);
+
+ val workerRemovealSuccessful = availableWorkers.remove(workerId);
+ Preconditions.checkState(workerRemovealSuccessful);
+
+ List<FunctionMetaData> allFunctions =
functionMetaDataManager.getAllFunctionMetaData();
+ Map<String, Function.Instance> allInstances =
+ computeAllInstances(allFunctions,
functionRuntimeManager.getRuntimeFactory().externallyManaged());
+
+ // Pretend that current assignments is empty, and get the
instances to assign.
+ Pair<List<Function.Instance>, List<Assignment>> instancesToAssign
+ = getUnassignedFunctionInstances(null, allInstances);
+
+ workerStatsManager.drainTotalExecTimeStart();
+ // Try to schedule the instances on the workers remaining
available after "workerId" is removed.
+ try {
+ postDrainAssignments =
scheduler.schedule(instancesToAssign.getLeft(), null, availableWorkers);
+ } catch (Exception e) {
+ log.info("invokeDrain: Got exception from schedule: ", e);
+ }
+ workerStatsManager.drainTotalExecTimeEnd();
+
+ if (postDrainAssignments != null) {
+ // log.info("drain: postDrainAssignments={}",
postDrainAssignments);
Review comment:
can we remove this commented out comment?
##########
File path:
pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/SchedulerManager.java
##########
@@ -403,6 +548,82 @@ private void scheduleCompaction(ScheduledExecutorService
executor, long schedule
}
}
+ @VisibleForTesting
+ List<Assignment> getCurrentPostDrainAssignments() {
+ return currentPostDrainAssignments;
+ }
+
+ @VisibleForTesting
+ void clearCurrentPostDrainAssignments() {
+ currentPostDrainAssignments = null;
+ }
+
+ @VisibleForTesting
+ List<Assignment> invokeDrain(String workerId) {
+
+ long startTime = System.nanoTime();
+
+ Set<String> availableWorkers = getCurrentAvailableWorkers();
+
+ // workerIdToAssignments is a map of the form {workerId :
{FullyQualifiedInstanceId : Assignment}}
+ Map<String, Map<String, Assignment>> workerIdToAssignments =
functionRuntimeManager.getCurrentAssignments();
+
+ // initialize stats collection
+ SchedulerStats schedulerStats = new
SchedulerStats(workerIdToAssignments, availableWorkers);
+
+ boolean drainSuccessful = false;
+ List<Assignment> postDrainAssignments = null;
+
+ try {
+ // The worker was checked to be among the available ones, by the
caller.
+ Preconditions.checkState(availableWorkers.contains(workerId));
+
+ drainOpStatusMap.put(workerId, DrainOpStatus.DrainInProgress);
+
+ val workerRemovealSuccessful = availableWorkers.remove(workerId);
+ Preconditions.checkState(workerRemovealSuccessful);
+
+ List<FunctionMetaData> allFunctions =
functionMetaDataManager.getAllFunctionMetaData();
+ Map<String, Function.Instance> allInstances =
+ computeAllInstances(allFunctions,
functionRuntimeManager.getRuntimeFactory().externallyManaged());
+
+ // Pretend that current assignments is empty, and get the
instances to assign.
+ Pair<List<Function.Instance>, List<Assignment>> instancesToAssign
+ = getUnassignedFunctionInstances(null, allInstances);
+
+ workerStatsManager.drainTotalExecTimeStart();
+ // Try to schedule the instances on the workers remaining
available after "workerId" is removed.
+ try {
+ postDrainAssignments =
scheduler.schedule(instancesToAssign.getLeft(), null, availableWorkers);
+ } catch (Exception e) {
+ log.info("invokeDrain: Got exception from schedule: ", e);
+ }
+ workerStatsManager.drainTotalExecTimeEnd();
+
+ if (postDrainAssignments != null) {
+ // log.info("drain: postDrainAssignments={}",
postDrainAssignments);
+ for (Assignment assignment : postDrainAssignments) {
+ MessageId messageId = publishNewAssignment(assignment,
false);
+ // Directly update in memory assignment cache since I am
leader
+ // log.info("Drain worker {} - new assignment: {}",
workerId, assignment);
Review comment:
same here
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
