arunpandianp commented on code in PR #38988:
URL: https://github.com/apache/beam/pull/38988#discussion_r3440482876
##########
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/SimpleParDoFnHelpers.java:
##########
@@ -293,23 +329,24 @@ public abstract void processTimer(
TimerInternals.TimerData timer,
Supplier<StreamingSideInputProcessor<?, ?>> sideInputProcessor)
throws Exception;
- };
+ }
void processTimers(
TimerType mode,
DataflowExecutionContext.DataflowStepContext context,
Coder<BoundedWindow> windowCoder,
- Runnable startKey,
Supplier<StreamingSideInputProcessor<?, ?>> sideInputProcessor)
throws Exception {
TimerInternals.TimerData timer = context.getNextFiredTimer(windowCoder);
-
if (timer != null && fnRunner == null) {
// If we need to run reallyStartBundle in here, we need to make sure to
switch the state
// sampler into the start state.
try (Closeable start = operationContext.enterStart()) {
reallyStartBundle();
- startKey.run();
+ }
+
+ if (!activeKey) {
+ this.onStartKey.accept((K) context.stateInternals().getKey());
Review Comment:
move out of ` if (timer != null && fnRunner == null)`
##########
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingSideInputDoFnRunner.java:
##########
@@ -38,6 +38,7 @@ public class StreamingSideInputDoFnRunner<InputT, OutputT, W
extends BoundedWind
implements DoFnRunner<InputT, OutputT> {
private final DoFnRunner<InputT, OutputT> simpleDoFnRunner;
private final StreamingSideInputProcessor<InputT, W> sideInputProcessor;
+ boolean activeKey = false;
Review Comment:
ping.
##########
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/PartialGroupByKeyParDoFns.java:
##########
@@ -378,15 +395,29 @@ public void processElement(Object elem) throws Exception {
}
@Override
- public void processTimers() {}
+ public void processTimers() throws Exception {
+ if (!activeKey) {
+ onStartKey();
+ }
+ }
@Override
- public void finishKey(Object key) throws Exception {}
+ public void finishKey(Object key) throws Exception {
+ if (!activeKey) {
+ onStartKey();
+ }
+ sideInputFetcher.persist();
+ sideInputFetcher = null;
+ this.activeKey = false;
+ this.sideInputFetcher = null;
+ }
@Override
public void finishBundle() throws Exception {
groupingTable.flush(receiver);
- sideInputFetcher.persist();
+ if (sideInputFetcher != null) {
Review Comment:
nit: can be removed
##########
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/SimpleParDoFnHelpers.java:
##########
@@ -233,18 +243,29 @@ public <TagT> void output(TupleTag<TagT> tag,
WindowedValue<TagT> output) {
fnRunner.startBundle();
}
+ void finishKey(StreamingSideInputProcessor<?, ?> sideInputProcessor) {
+ if (!activeKey) {
+ // This means that there were no elements for this key. Try to unblock
any queued elements.
+ onStartKey.accept((K) stepContext.stateInternals().getKey());
+ }
+ if (sideInputProcessor != null) {
+ sideInputProcessor.handleFinishKeyOrBundle();
+ }
+ this.activeKey = false;
+ }
+
void finishBundle(StreamingSideInputProcessor<?, ?> sideInputProcessor)
throws Exception {
if (fnRunner != null) {
fnRunner.finishBundle();
if (sideInputProcessor != null) {
Review Comment:
do we need the`sideInputProcessor.handleFinishKeyOrBundle` call here?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
