This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to tag nightly-master
in repository https://gitbox.apache.org/repos/asf/beam.git.
*** WARNING: tag nightly-master was modified! ***
from 39cf3fc (commit)
to 10e3e57 (commit)
from 39cf3fc Merge pull request #15264 from [BEAM-12670] Relocate bq
client exception imports to try block and conditionally turn off tests if
imports fail
add c4508c8 [BEAM-12591] Put Spark Structured Streaming runner sources
back to main src folder
add 473d187 [BEAM-12629] As spark DataSourceV2 is only available for
spark 2, provide a DataSourceV2 based impl for spark 2 and create a structure
for extension with a spark 3 source.
add ad6bea8 [BEAM-12627] Deal with spark Encoders braking change between
spark 2 and spark 3 by providing an implementation for each of them.
add f0014d9 [BEAM-12591] move SchemaHelpers to correct package
add fd9bb74 [BEAM-8470] Disable wait for termination in a streaming
pipeline because it is infinite by definition
add 94ce5d3 [BEAM-12630] Deal with breaking change in streaming pipelines
start by introducing an AbstractTranslationContext and version specific
implementations
add b8dc86c [BEAM-12629] Make source tests spark version agnostic and
move them back to common spark module
add b1d5dc4 [BEAM-12629] Make a spark 3 source impl
add 75247cb [BEAM-12591] Fix checkstyle and spotless
add e10b2eb [BEAM-12629] Reduce serializable to only needed classes and
Fix schema inference
add cc3ff98 [BEAM-12591] Add checkstyle exceptions for version specific
classes because checkstyle does not correctly detect package files across
multiple source directories
add 81033b1 [BEAM-12629] Fix sources javadocs and improve impl
add 23fd65d [BEAM-12591] Add spark 3 to structured streaming validates
runner tests
add 2144cab Merge pull request #15218 from
echauchot/BEAM-7093-spark3-fix-for-SS-runner
add 291aa6c [BEAM-6516] Fixes race condition in RabbitMqIO causing
duplicate acks (#15157)
add 7bfeca2 [BEAM-12601] Add append-only option (#15257)
add 57b6d77 Revert "[BEAM-11934] Remove Dataflow override of streaming
WriteFiles with runner determined sharding (#15178)"
add f759a5c Revert "[BEAM-11934] Remove Dataflow override of streaming
WriteFiles with runner determined sharding"
add 0ac5480 Add google cloud heap profiling support to beam java sdk
container
add 10e3e57 Merge pull request #15288 from andyxiexu/java-sdk
No new revisions were added by this update.
Summary of changes:
...ValidatesRunner_SparkStructuredStreaming.groovy | 1 +
CHANGES.md | 2 +
.../beam/runners/dataflow/DataflowRunner.java | 73 +++++++
.../beam/runners/dataflow/DataflowRunnerTest.java | 120 +++++------
.../translation/TranslationContext.java | 240 +--------------------
.../translation/batch/DatasetSourceBatch.java | 9 +-
.../translation/helpers/EncoderFactory.java | 54 +++++
.../streaming/DatasetSourceStreaming.java | 9 +-
.../translation/batch/SimpleSourceTest.java | 101 ---------
.../translation/TranslationContext.java} | 29 ++-
.../translation/batch/DatasetSourceBatch.java | 240 +++++++++++++++++++++
.../translation/helpers/EncoderFactory.java | 49 +++++
.../streaming/DatasetSourceStreaming.java} | 9 +-
.../spark/structuredstreaming/Constants.java} | 8 +-
.../SparkStructuredStreamingPipelineOptions.java | 0
.../SparkStructuredStreamingPipelineResult.java | 0
.../SparkStructuredStreamingRunner.java | 6 +-
.../SparkStructuredStreamingRunnerRegistrar.java | 0
.../aggregators/AggregatorsAccumulator.java | 0
.../aggregators/NamedAggregators.java | 0
.../aggregators/NamedAggregatorsAccumulator.java | 0
.../aggregators/package-info.java | 0
.../structuredstreaming/examples/WordCount.java | 0
.../metrics/AggregatorMetric.java | 0
.../metrics/AggregatorMetricSource.java | 0
.../metrics/CompositeSource.java | 0
.../metrics/MetricsAccumulator.java | 0
.../MetricsContainerStepMapAccumulator.java | 0
.../metrics/SparkBeamMetric.java | 0
.../metrics/SparkBeamMetricSource.java | 0
.../metrics/SparkMetricsContainerStepMap.java | 0
.../metrics/WithMetricsSupport.java | 0
.../structuredstreaming/metrics/package-info.java | 0
.../metrics/sink/CodahaleCsvSink.java | 0
.../metrics/sink/CodahaleGraphiteSink.java | 0
.../metrics/sink/package-info.java | 0
.../spark/structuredstreaming/package-info.java | 0
.../translation/AbstractTranslationContext.java} | 19 +-
.../translation/PipelineTranslator.java | 4 +-
.../translation/SparkTransformOverrides.java | 0
.../translation/TransformTranslator.java | 2 +-
.../translation/batch/AggregatorCombiner.java | 0
.../batch/CombinePerKeyTranslatorBatch.java | 4 +-
.../CreatePCollectionViewTranslatorBatch.java | 4 +-
.../translation/batch/DoFnFunction.java | 0
.../translation/batch/DoFnRunnerWithMetrics.java | 0
.../translation/batch/FlattenTranslatorBatch.java | 4 +-
.../batch/GroupByKeyTranslatorBatch.java | 4 +-
.../translation/batch/ImpulseTranslatorBatch.java | 4 +-
.../translation/batch/ParDoTranslatorBatch.java | 14 +-
.../translation/batch/PipelineTranslatorBatch.java | 0
.../translation/batch/ProcessContext.java | 0
.../batch/ReadSourceTranslatorBatch.java | 14 +-
.../batch/ReshuffleTranslatorBatch.java | 4 +-
.../batch/WindowAssignTranslatorBatch.java | 4 +-
.../GroupAlsoByWindowViaOutputBufferFn.java | 0
.../batch/functions/NoOpStepContext.java | 0
.../batch/functions/SparkSideInputReader.java | 0
.../translation/batch/functions/package-info.java | 0
.../translation/batch/package-info.java | 0
.../translation/helpers/CoderHelpers.java | 0
.../translation/helpers/EncoderHelpers.java | 21 +-
.../translation/helpers/KVHelpers.java | 0
.../translation/helpers/MultiOutputCoder.java | 0
.../translation/helpers/RowHelpers.java | 0
.../translation/helpers}/SchemaHelpers.java | 2 +-
.../translation/helpers/SideInputBroadcast.java | 0
.../translation/helpers/WindowingHelpers.java | 4 +-
.../translation/helpers/package-info.java | 0
.../translation/package-info.java | 0
.../streaming/PipelineTranslatorStreaming.java | 0
.../streaming/ReadSourceTranslatorStreaming.java | 14 +-
.../translation/streaming/package-info.java | 0
.../translation/utils/CachedSideInputReader.java | 0
.../translation/utils/SideInputStorage.java | 0
.../translation/utils/package-info.java | 0
...parkStructuredStreamingRunnerRegistrarTest.java | 0
.../StructuredStreamingPipelineStateTest.java | 0
.../aggregators/metrics/sink/InMemoryMetrics.java | 0
.../metrics/sink/InMemoryMetricsSinkRule.java | 0
.../metrics/sink/SparkMetricsSinkTest.java | 0
.../metrics/BeamMetricTest.java | 0
.../translation/batch/CombineTest.java | 0
.../translation/batch/ComplexSourceTest.java | 0
.../translation/batch/FlattenTest.java | 0
.../translation/batch/GroupByKeyTest.java | 0
.../translation/batch/ParDoTest.java | 0
.../translation/batch}/SimpleSourceTest.java | 18 +-
.../translation/batch/WindowAssignTest.java | 0
.../translation/helpers/EncoderHelpersTest.java | 0
.../translation/streaming/SimpleSourceTest.java | 0
.../utils/SerializationDebugger.java | 0
.../structuredstreaming/utils/package-info.java | 0
.../src/main/resources/beam/suppressions.xml | 11 +
sdks/java/container/boot.go | 17 +-
.../main/java/org/apache/beam/sdk/io/FileIO.java | 9 +-
.../sdk/io/elasticsearch/ElasticsearchIOTest.java | 12 ++
.../sdk/io/elasticsearch/ElasticsearchIOTest.java | 12 ++
.../sdk/io/elasticsearch/ElasticsearchIOTest.java | 12 ++
.../elasticsearch/ElasticsearchIOTestCommon.java | 19 ++
.../beam/sdk/io/elasticsearch/ElasticsearchIO.java | 80 +++++--
.../apache/beam/sdk/io/rabbitmq/RabbitMqIO.java | 15 +-
102 files changed, 740 insertions(+), 537 deletions(-)
create mode 100644
runners/spark/2/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/EncoderFactory.java
delete mode 100644
runners/spark/2/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/SimpleSourceTest.java
copy
runners/spark/{2/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/NoOpStepContext.java
=>
3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TranslationContext.java}
(53%)
create mode 100644
runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/DatasetSourceBatch.java
create mode 100644
runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/EncoderFactory.java
copy
runners/spark/{2/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/package-info.java
=>
3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/DatasetSourceStreaming.java}
(74%)
copy
runners/spark/{2/src/main/java/org/apache/beam/runners/spark/structuredstreaming/package-info.java
=>
src/main/java/org/apache/beam/runners/spark/structuredstreaming/Constants.java}
(78%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingPipelineOptions.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingPipelineResult.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingRunner.java
(98%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingRunnerRegistrar.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/AggregatorsAccumulator.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/NamedAggregators.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/NamedAggregatorsAccumulator.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/package-info.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/examples/WordCount.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/AggregatorMetric.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/AggregatorMetricSource.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/CompositeSource.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/MetricsAccumulator.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/MetricsContainerStepMapAccumulator.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/SparkBeamMetric.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/SparkBeamMetricSource.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/SparkMetricsContainerStepMap.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/WithMetricsSupport.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/package-info.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/sink/CodahaleCsvSink.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/sink/CodahaleGraphiteSink.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/sink/package-info.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/package-info.java
(100%)
copy
runners/spark/{2/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TranslationContext.java
=>
src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/AbstractTranslationContext.java}
(94%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java
(98%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/SparkTransformOverrides.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java
(93%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/AggregatorCombiner.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CombinePerKeyTranslatorBatch.java
(98%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CreatePCollectionViewTranslatorBatch.java
(97%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/DoFnFunction.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/DoFnRunnerWithMetrics.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/FlattenTranslatorBatch.java
(97%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/GroupByKeyTranslatorBatch.java
(98%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ImpulseTranslatorBatch.java
(94%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ParDoTranslatorBatch.java
(96%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/PipelineTranslatorBatch.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ProcessContext.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ReadSourceTranslatorBatch.java
(85%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ReshuffleTranslatorBatch.java
(94%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/WindowAssignTranslatorBatch.java
(95%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/GroupAlsoByWindowViaOutputBufferFn.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/NoOpStepContext.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/SparkSideInputReader.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/package-info.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/package-info.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/CoderHelpers.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/EncoderHelpers.java
(89%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/KVHelpers.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/MultiOutputCoder.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/RowHelpers.java
(100%)
rename
runners/spark/{2/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation
=>
src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers}/SchemaHelpers.java
(99%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/SideInputBroadcast.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/WindowingHelpers.java
(97%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/package-info.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/package-info.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/PipelineTranslatorStreaming.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/ReadSourceTranslatorStreaming.java
(87%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/package-info.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/utils/CachedSideInputReader.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/utils/SideInputStorage.java
(100%)
rename runners/spark/{2 =>
}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/utils/package-info.java
(100%)
rename runners/spark/{2 =>
}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingRunnerRegistrarTest.java
(100%)
rename runners/spark/{2 =>
}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/StructuredStreamingPipelineStateTest.java
(100%)
rename runners/spark/{2 =>
}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/metrics/sink/InMemoryMetrics.java
(100%)
rename runners/spark/{2 =>
}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/metrics/sink/InMemoryMetricsSinkRule.java
(100%)
rename runners/spark/{2 =>
}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/metrics/sink/SparkMetricsSinkTest.java
(100%)
rename runners/spark/{2 =>
}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/metrics/BeamMetricTest.java
(100%)
rename runners/spark/{2 =>
}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CombineTest.java
(100%)
rename runners/spark/{2 =>
}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ComplexSourceTest.java
(100%)
rename runners/spark/{2 =>
}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/FlattenTest.java
(100%)
rename runners/spark/{2 =>
}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/GroupByKeyTest.java
(100%)
rename runners/spark/{2 =>
}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ParDoTest.java
(100%)
copy
runners/spark/{2/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming
=>
src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch}/SimpleSourceTest.java
(79%)
rename runners/spark/{2 =>
}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/WindowAssignTest.java
(100%)
rename runners/spark/{2 =>
}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/EncoderHelpersTest.java
(100%)
rename runners/spark/{2 =>
}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/SimpleSourceTest.java
(100%)
rename runners/spark/{2 =>
}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/utils/SerializationDebugger.java
(100%)
rename runners/spark/{2 =>
}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/utils/package-info.java
(100%)
