This is an automated email from the ASF dual-hosted git repository.
aromanenko pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
from fd5ad1ec633 Reduce version matrix of build wheel on pull request event
(#24448)
add 46919b7bbf7 [Spark dataset runner] Make sure PCollection views get
only broadcasted once if reused (#24288)
No new revisions were added by this update.
Summary of changes:
.../translation/PipelineTranslator.java | 27 ++-
.../translation/SparkSessionFactory.java | 148 ++++++++++++++++
.../translation/TransformTranslator.java | 24 ++-
.../batch/DoFnMapPartitionsFactory.java | 31 +---
.../translation/batch/ParDoTranslatorBatch.java | 99 ++++++-----
.../batch/functions/CachedSideInputReader.java | 176 +++++++++++++++++++
.../batch/functions/SideInputValues.java | 189 ++++++++++++++++++++
.../batch/functions/SparkSideInputReader.java | 193 ++++++++-------------
.../translation/helpers/SideInputBroadcast.java | 49 ------
.../translation/utils/CachedSideInputReader.java | 93 ----------
.../translation/utils/SideInputStorage.java | 107 ------------
.../batch/functions/SideInputValuesTest.java | 129 ++++++++++++++
runners/spark/spark_runner.gradle | 8 +
13 files changed, 825 insertions(+), 448 deletions(-)
create mode 100644
runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/CachedSideInputReader.java
create mode 100644
runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/SideInputValues.java
delete mode 100644
runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/SideInputBroadcast.java
delete mode 100644
runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/utils/CachedSideInputReader.java
delete mode 100644
runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/utils/SideInputStorage.java
create mode 100644
runners/spark/3/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/SideInputValuesTest.java
