This is an automated email from the ASF dual-hosted git repository.
yhu pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
from 9628c1ef9ba [KafkaIO] Improve caching in backlog estimation and
processing (#34331)
add 74c514fc392 Spark Runner : Support for Streaming side-inputs for Spark
Runner (#34560)
No new revisions were added by this update.
Summary of changes:
...beam_PostCommit_Java_ValidatesRunner_Spark.json | 3 +-
...stCommit_Java_ValidatesRunner_Spark_Java11.json | 3 +-
CHANGES.md | 1 +
runners/spark/spark_runner.gradle | 1 -
.../runners/spark/SparkTransformOverrides.java | 17 ++
.../beam/runners/spark/coders/CoderHelpers.java | 18 ++
.../spark/translation/EvaluationContext.java | 28 +++
.../spark/translation/MultiDoFnFunction.java | 10 +-
.../spark/translation/SideInputMetadata.java | 70 ++++++++
.../spark/translation/SparkPCollectionView.java | 37 +++-
.../spark/translation/TransformTranslator.java | 4 +-
.../streaming/CreateStreamingSparkView.java} | 43 ++---
.../translation/streaming/ParDoStateUpdateFn.java | 11 +-
.../streaming/StatefulStreamingParDoEvaluator.java | 3 +-
.../streaming/StreamingTransformTranslator.java | 79 +++++++-
.../runners/spark/util/SideInputBroadcast.java | 15 +-
.../runners/spark/util/SideInputReaderFactory.java | 52 ++++++
.../runners/spark/util/SparkSideInputReader.java | 57 +++++-
.../StreamingTransformTranslatorTest.java | 200 ++++++++++++++++++---
19 files changed, 576 insertions(+), 76 deletions(-)
create mode 100644
runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SideInputMetadata.java
copy
runners/{flink/src/main/java/org/apache/beam/runners/flink/CreateStreamingFlinkView.java
=>
spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/CreateStreamingSparkView.java}
(79%)
create mode 100644
runners/spark/src/main/java/org/apache/beam/runners/spark/util/SideInputReaderFactory.java
