This is an automated email from the ASF dual-hosted git repository.
echauchot pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git.
from 39cf3fc Merge pull request #15264 from [BEAM-12670] Relocate bq
client exception imports to try block and conditionally turn off tests if
imports fail
new c4508c8 [BEAM-12591] Put Spark Structured Streaming runner sources
back to main src folder
new 473d187 [BEAM-12629] As spark DataSourceV2 is only available for
spark 2, provide a DataSourceV2 based impl for spark 2 and create a structure
for extension with a spark 3 source.
new ad6bea8 [BEAM-12627] Deal with spark Encoders braking change between
spark 2 and spark 3 by providing an implementation for each of them.
new f0014d9 [BEAM-12591] move SchemaHelpers to correct package
new fd9bb74 [BEAM-8470] Disable wait for termination in a streaming
pipeline because it is infinite by definition
new 94ce5d3 [BEAM-12630] Deal with breaking change in streaming pipelines
start by introducing an AbstractTranslationContext and version specific
implementations
new b8dc86c [BEAM-12629] Make source tests spark version agnostic and
move them back to common spark module
new b1d5dc4 [BEAM-12629] Make a spark 3 source impl
new 75247cb [BEAM-12591] Fix checkstyle and spotless
new e10b2eb [BEAM-12629] Reduce serializable to only needed classes and
Fix schema inference
new cc3ff98 [BEAM-12591] Add checkstyle exceptions for version specific
classes because checkstyle does not correctly detect package files across
multiple source directories
new 81033b1 [BEAM-12629] Fix sources javadocs and improve impl
new 23fd65d [BEAM-12591] Add spark 3 to structured streaming validates
runner tests
new 2144cab Merge pull request #15218 from
echauchot/BEAM-7093-spark3-fix-for-SS-runner
The 32603 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
...ValidatesRunner_SparkStructuredStreaming.groovy | 1 +
.../translation/TranslationContext.java | 240 +--------------------
.../translation/batch/DatasetSourceBatch.java | 9 +-
.../translation/helpers/EncoderFactory.java | 54 +++++
.../streaming/DatasetSourceStreaming.java | 9 +-
.../translation/batch/SimpleSourceTest.java | 101 ---------
.../translation/TranslationContext.java} | 29 ++-
.../translation/batch/DatasetSourceBatch.java | 240 +++++++++++++++++++++
.../translation/helpers/EncoderFactory.java | 49 +++++
.../streaming/DatasetSourceStreaming.java} | 9 +-
.../spark/structuredstreaming/Constants.java} | 8 +-
.../SparkStructuredStreamingPipelineOptions.java | 0
.../SparkStructuredStreamingPipelineResult.java | 0
.../SparkStructuredStreamingRunner.java | 6 +-
.../SparkStructuredStreamingRunnerRegistrar.java | 0
.../aggregators/AggregatorsAccumulator.java | 0
.../aggregators/NamedAggregators.java | 0
.../aggregators/NamedAggregatorsAccumulator.java | 0
.../aggregators/package-info.java | 0
.../structuredstreaming/examples/WordCount.java | 0
.../metrics/AggregatorMetric.java | 0
.../metrics/AggregatorMetricSource.java | 0
.../metrics/CompositeSource.java | 0
.../metrics/MetricsAccumulator.java | 0
.../MetricsContainerStepMapAccumulator.java | 0
.../metrics/SparkBeamMetric.java | 0
.../metrics/SparkBeamMetricSource.java | 0
.../metrics/SparkMetricsContainerStepMap.java | 0
.../metrics/WithMetricsSupport.java | 0
.../structuredstreaming/metrics/package-info.java | 0
.../metrics/sink/CodahaleCsvSink.java | 0
.../metrics/sink/CodahaleGraphiteSink.java | 0
.../metrics/sink/package-info.java | 0
.../spark/structuredstreaming/package-info.java | 0
.../translation/AbstractTranslationContext.java} | 19 +-
.../translation/PipelineTranslator.java | 4 +-
.../translation/SparkTransformOverrides.java | 0
.../translation/TransformTranslator.java | 2 +-
.../translation/batch/AggregatorCombiner.java | 0
.../batch/CombinePerKeyTranslatorBatch.java | 4 +-
.../CreatePCollectionViewTranslatorBatch.java | 4 +-
.../translation/batch/DoFnFunction.java | 0
.../translation/batch/DoFnRunnerWithMetrics.java | 0
.../translation/batch/FlattenTranslatorBatch.java | 4 +-
.../batch/GroupByKeyTranslatorBatch.java | 4 +-
.../translation/batch/ImpulseTranslatorBatch.java | 4 +-
.../translation/batch/ParDoTranslatorBatch.java | 14 +-
.../translation/batch/PipelineTranslatorBatch.java | 0
.../translation/batch/ProcessContext.java | 0
.../batch/ReadSourceTranslatorBatch.java | 14 +-
.../batch/ReshuffleTranslatorBatch.java | 4 +-
.../batch/WindowAssignTranslatorBatch.java | 4 +-
.../GroupAlsoByWindowViaOutputBufferFn.java | 0
.../batch/functions/NoOpStepContext.java | 0
.../batch/functions/SparkSideInputReader.java | 0
.../translation/batch/functions/package-info.java | 0
.../translation/batch/package-info.java | 0
.../translation/helpers/CoderHelpers.java | 0
.../translation/helpers/EncoderHelpers.java | 21 +-
.../translation/helpers/KVHelpers.java | 0
.../translation/helpers/MultiOutputCoder.java | 0
.../translation/helpers/RowHelpers.java | 0
.../translation/helpers}/SchemaHelpers.java | 2 +-
.../translation/helpers/SideInputBroadcast.java | 0
.../translation/helpers/WindowingHelpers.java | 4 +-
.../translation/helpers/package-info.java | 0
.../translation/package-info.java | 0
.../streaming/PipelineTranslatorStreaming.java | 0
.../streaming/ReadSourceTranslatorStreaming.java | 14 +-
.../translation/streaming/package-info.java | 0
.../translation/utils/CachedSideInputReader.java | 0
.../translation/utils/SideInputStorage.java | 0
.../translation/utils/package-info.java | 0
...parkStructuredStreamingRunnerRegistrarTest.java | 0
.../StructuredStreamingPipelineStateTest.java | 0
.../aggregators/metrics/sink/InMemoryMetrics.java | 0
.../metrics/sink/InMemoryMetricsSinkRule.java | 0
.../metrics/sink/SparkMetricsSinkTest.java | 0
.../metrics/BeamMetricTest.java | 0
.../translation/batch/CombineTest.java | 0
.../translation/batch/ComplexSourceTest.java | 0
.../translation/batch/FlattenTest.java | 0
.../translation/batch/GroupByKeyTest.java | 0
.../translation/batch/ParDoTest.java | 0
.../translation/batch}/SimpleSourceTest.java | 18 +-
.../translation/batch/WindowAssignTest.java | 0
.../translation/helpers/EncoderHelpersTest.java | 0
.../translation/streaming/SimpleSourceTest.java | 0
.../utils/SerializationDebugger.java | 0
.../structuredstreaming/utils/package-info.java | 0
.../src/main/resources/beam/suppressions.xml | 11 +
91 files changed, 464 insertions(+), 442 deletions(-)
create mode 100644
runners/spark/2/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/EncoderFactory.java
delete mode 100644
runners/spark/2/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/SimpleSourceTest.java
copy
runners/spark/{2/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/NoOpStepContext.java
=>
3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TranslationContext.java}
(53%)
create mode 100644
runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/DatasetSourceBatch.java
create mode 100644
runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/EncoderFactory.java
copy
runners/spark/{2/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/package-info.java
=>
3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/DatasetSourceStreaming.java}
(74%)
copy
runners/spark/{2/src/main/java/org/apache/beam/runners/spark/structuredstreaming/package-info.java
=>
src/main/java/org/apache/beam/runners/spark/structuredstreaming/Constants.java}
(78%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingPipelineOptions.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingPipelineResult.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingRunner.java
(98%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingRunnerRegistrar.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/AggregatorsAccumulator.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/NamedAggregators.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/NamedAggregatorsAccumulator.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/package-info.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/examples/WordCount.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/AggregatorMetric.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/AggregatorMetricSource.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/CompositeSource.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/MetricsAccumulator.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/MetricsContainerStepMapAccumulator.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/SparkBeamMetric.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/SparkBeamMetricSource.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/SparkMetricsContainerStepMap.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/WithMetricsSupport.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/package-info.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/sink/CodahaleCsvSink.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/sink/CodahaleGraphiteSink.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/sink/package-info.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/package-info.java
(100%)
copy
runners/spark/{2/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TranslationContext.java
=>
src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/AbstractTranslationContext.java}
(94%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java
(98%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/SparkTransformOverrides.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java
(93%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/AggregatorCombiner.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CombinePerKeyTranslatorBatch.java
(98%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CreatePCollectionViewTranslatorBatch.java
(97%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/DoFnFunction.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/DoFnRunnerWithMetrics.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/FlattenTranslatorBatch.java
(97%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/GroupByKeyTranslatorBatch.java
(98%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ImpulseTranslatorBatch.java
(94%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ParDoTranslatorBatch.java
(96%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/PipelineTranslatorBatch.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ProcessContext.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ReadSourceTranslatorBatch.java
(85%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ReshuffleTranslatorBatch.java
(94%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/WindowAssignTranslatorBatch.java
(95%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/GroupAlsoByWindowViaOutputBufferFn.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/NoOpStepContext.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/SparkSideInputReader.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/package-info.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/package-info.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/CoderHelpers.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/EncoderHelpers.java
(89%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/KVHelpers.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/MultiOutputCoder.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/RowHelpers.java
(100%)
rename
runners/spark/{2/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation
=>
src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers}/SchemaHelpers.java
(99%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/SideInputBroadcast.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/WindowingHelpers.java
(97%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/package-info.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/package-info.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/PipelineTranslatorStreaming.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/ReadSourceTranslatorStreaming.java
(87%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/package-info.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/utils/CachedSideInputReader.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/utils/SideInputStorage.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/utils/package-info.java
(100%)
rename runners/spark/{2 =>
}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingRunnerRegistrarTest.java
(100%)
rename runners/spark/{2 =>
}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/StructuredStreamingPipelineStateTest.java
(100%)
rename runners/spark/{2 =>
}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/metrics/sink/InMemoryMetrics.java
(100%)
rename runners/spark/{2 =>
}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/metrics/sink/InMemoryMetricsSinkRule.java
(100%)
rename runners/spark/{2 =>
}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/metrics/sink/SparkMetricsSinkTest.java
(100%)
rename runners/spark/{2 =>
}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/metrics/BeamMetricTest.java
(100%)
rename runners/spark/{2 =>
}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CombineTest.java
(100%)
rename runners/spark/{2 =>
}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ComplexSourceTest.java
(100%)
rename runners/spark/{2 =>
}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/FlattenTest.java
(100%)
rename runners/spark/{2 =>
}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/GroupByKeyTest.java
(100%)
rename runners/spark/{2 =>
}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ParDoTest.java
(100%)
copy
runners/spark/{2/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming
=>
src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch}/SimpleSourceTest.java
(79%)
rename runners/spark/{2 =>
}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/WindowAssignTest.java
(100%)
rename runners/spark/{2 =>
}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/EncoderHelpersTest.java
(100%)
rename runners/spark/{2 =>
}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/SimpleSourceTest.java
(100%)
rename runners/spark/{2 =>
}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/utils/SerializationDebugger.java
(100%)
rename runners/spark/{2 =>
}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/utils/package-info.java
(100%)
