m-trieu commented on code in PR #31883:
URL: https://github.com/apache/beam/pull/31883#discussion_r1702426619
##########
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/client/grpc/GrpcDirectGetWorkStream.java:
##########
@@ -240,25 +234,46 @@ public void sendHealthCheck() {
@Override
protected void onResponse(StreamingGetWorkResponseChunk chunk) {
getWorkThrottleTimer.stop();
- WorkItemBuffer workItemBuffer =
- workItemBuffers.computeIfAbsent(chunk.getStreamId(), unused -> new
WorkItemBuffer());
+ GetWorkItemBuffer workItemBuffer =
+ workItemBuffers.computeIfAbsent(chunk.getStreamId(), unused -> new
GetWorkItemBuffer());
workItemBuffer.append(chunk);
// The entire WorkItem has been received, it is ready to be processed.
if (chunk.getRemainingBytesForWorkItem() == 0) {
- workItemBuffer.runAndReset();
+ long size = workItemBuffer.bufferedSize();
+ workItemBuffer.flushToWorkItem().ifPresent(this::consumeWorkItem);
// Record the fact that there are now fewer outstanding messages and
bytes on the stream.
- inFlightBudget.updateAndGet(budget -> budget.subtract(1,
workItemBuffer.bufferedSize()));
+ inFlightBudget.updateAndGet(budget -> budget.subtract(1, size));
}
}
+ private void consumeWorkItem(ConstructedWorkItem constructedWorkItem) {
+ WorkItem workItem = constructedWorkItem.workItem();
+ GetWorkItemBuffer.ComputationMetadata metadata =
constructedWorkItem.computationMetadata();
+ pendingResponseBudget.getAndUpdate(budget -> budget.apply(1,
workItem.getSerializedSize()));
+ try {
+ workItemScheduler.scheduleWork(
+ workItem,
+ createWatermarks(workItem, Preconditions.checkNotNull(metadata)),
+
createProcessingContext(Preconditions.checkNotNull(metadata.computationId())),
+ constructedWorkItem.latencyAttributions());
+ } finally {
+ pendingResponseBudget.getAndUpdate(budget -> budget.apply(-1,
-workItem.getSerializedSize()));
+ }
+ }
+
+ private Work.ProcessingContext createProcessingContext(String computationId)
{
+ return Work.createProcessingContext(
+ computationId, getDataClient.get(), workCommitter.get()::commit,
heartbeatSender.get());
+ }
+
@Override
protected void startThrottleTimer() {
getWorkThrottleTimer.start();
}
@Override
- public synchronized void adjustBudget(long itemsDelta, long bytesDelta) {
+ public void adjustBudget(long itemsDelta, long bytesDelta) {
nextBudgetAdjustment.set(nextBudgetAdjustment.get().apply(itemsDelta,
bytesDelta));
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
