arunpandianp commented on code in PR #40593:
URL: https://github.com/apache/beam/pull/40593#discussion_r4296946044
##########
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/BoundedQueueExecutor.java:
##########
@@ -403,25 +501,52 @@ BoundedQueueExecutorWorkHandleImpl
createBudgetHandle(Work work, long bytes) {
checkArgument(handle instanceof BoundedQueueExecutorWorkHandleImpl);
checkStateNotNull(onFailedWorkHandler);
BoundedQueueExecutorWorkHandleImpl internalHandle =
(BoundedQueueExecutorWorkHandleImpl) handle;
- if (keyGroupWorkQueue == null) {
+ @Nullable KeyGroupWorkIndex index = keyGroupWorkIndex;
+ if (index == null) {
return null;
}
while (true) {
- @Nullable QueuedWork queuedWork =
keyGroupWorkQueue.pollWork(computationId, keyGroup);
- if (queuedWork == null) {
+ @Nullable QueuedWork stolen = index.pollAndClaim(computationId,
keyGroup);
+ if (stolen == null) {
return null;
}
- Work work = queuedWork.getWork().work();
+ ExecutableWork stolenWork = stolen.getWork();
+ BoundedQueueExecutorWorkHandleImpl stolenHandle = stolen.getHandle();
+ stolen.clearClaimedWork();
+ onWorkStolen();
+ Work work = stolenWork.work();
if (work.isFailed()) {
- queuedWork.getHandle().close();
+ stolenHandle.close();
onFailedWorkHandler.onFailedWork(work);
continue;
}
- internalHandle.merge(queuedWork.getHandle());
- return queuedWork.getWork();
+ internalHandle.merge(stolenHandle);
+ return stolenWork;
}
}
+ /**
+ * Purges stale shells in bulk if they accumulate while all executor threads
are busy running
+ * multi-key bundles.
+ */
+ private void onWorkStolen() {
+ if (workQueue.size() <= (long) elementsOutstanding.get() +
maximumElementsOutstanding
Review Comment:
Updated javadoc.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
