This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from 511036ad997 Revise CHANGES.md for upcoming release (#37243)
add bfff80b6fa4 [Dataflow Streaming] Add WindmillTagEncodingV2. (#37151)
add be924b74637 Update BeamModulePlugin.groovy (#37233)
add 6bedec369f7 Bump cloud.google.com/go/bigtable from 1.40.1 to 1.41.0 in
/sdks (#37120)
add 7694450ecc2 Fix CONNECTION_STRING format in tox.ini (#37250)
add d7fa511ee8a [yaml] : add more error handling tests and website example
(#37245)
add ef0a03c90b2 Flink 2.0 support (#37175)
No new revisions were added by this update.
Summary of changes:
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 5 +-
gradle.properties | 2 +-
.../apache/beam/runners/core/StateNamespaces.java | 8 +
.../org/apache/beam/runners/core/StateTags.java | 17 +
runners/flink/2.0/build.gradle | 43 ++
.../job-server-container/build.gradle | 0
.../flink/{1.19 => 2.0}/job-server/build.gradle | 2 +-
.../flink/FlinkBatchTranslationContext.java | 58 +--
.../runners/flink/FlinkExecutionEnvironments.java | 189 +++----
.../runners/flink/FlinkMiniClusterEntryPoint.java | 4 +-
.../flink/FlinkPipelineExecutionEnvironment.java | 89 +---
.../beam/runners/flink/FlinkPipelineOptions.java | 49 +-
.../beam/runners/flink/FlinkPipelineRunner.java | 11 +-
.../runners/flink/FlinkPipelineTranslator.java | 10 +-
.../FlinkStreamingPortablePipelineTranslator.java | 4 +-
.../flink/FlinkStreamingTransformTranslators.java | 17 +-
.../runners/flink/FlinkTransformOverrides.java | 10 +-
.../translation/functions/FlinkDoFnFunction.java | 4 +-
.../FlinkExecutableStageContextFactory.java | 4 +-
.../functions/FlinkExecutableStageFunction.java | 4 +-
.../FlinkExecutableStagePruningFunction.java | 4 +-
.../FlinkMergingNonShuffleReduceFunction.java | 4 +-
.../functions/FlinkMultiOutputPruningFunction.java | 4 +-
.../functions/FlinkPartialReduceFunction.java | 4 +-
.../translation/functions/FlinkReduceFunction.java | 4 +-
.../functions/FlinkStatefulDoFnFunction.java | 4 +-
.../functions/ImpulseSourceFunction.java | 2 +-
.../translation/types/CoderTypeInformation.java | 3 +-
.../translation/types/EncodedValueSerializer.java | 13 +-
.../types/EncodedValueTypeInformation.java | 3 +-
.../types/UnversionedTypeSerializerSnapshot.java | 3 +-
.../wrappers/streaming/DoFnOperator.java | 4 +-
.../streaming/io/StreamingImpulseSource.java | 10 +-
.../wrappers/streaming/io/TestStreamSource.java | 2 +-
.../streaming/io/UnboundedSourceWrapper.java | 23 +-
.../state/FlinkBroadcastStateInternals.java | 4 +-
.../streaming/state/FlinkStateInternals.java | 2 +-
.../runners/flink/EncodedValueComparatorTest.java | 3 +-
.../flink/FlinkExecutionEnvironmentsTest.java | 60 ++-
.../FlinkPipelineExecutionEnvironmentTest.java | 44 +-
.../runners/flink/FlinkPipelineOptionsTest.java | 9 +-
.../flink/FlinkRequiresStableInputTest.java | 10 +-
.../apache/beam/runners/flink/FlinkRunnerTest.java | 9 +-
.../beam/runners/flink/FlinkSavepointTest.java | 9 +-
.../beam/runners/flink/FlinkSubmissionTest.java | 42 +-
.../apache/beam/runners/flink/ReadSourceTest.java | 17 +-
.../adapter/BeamFlinkDataStreamAdapterTest.java | 6 +-
.../flink/streaming/BoundedSourceRestoreTest.java | 8 +-
.../flink/streaming/FlinkStateInternalsTest.java | 4 +-
.../flink/streaming/MemoryStateBackendWrapper.java | 10 +-
.../runners/flink/streaming/StreamSources.java | 2 +-
.../functions/FlinkDoFnFunctionTest.java | 4 +-
.../FlinkExecutableStageFunctionTest.java | 12 +-
.../functions/FlinkStatefulDoFnFunctionTest.java | 4 +-
.../functions/ImpulseSourceFunctionTest.java | 2 +-
.../streaming/io/UnboundedSourceWrapperTest.java | 72 ++-
.../stableinput/BufferingDoFnRunnerTest.java | 6 +-
.../2.0/src/test/resources/flink-test-config.yaml | 16 +-
runners/flink/flink_runner.gradle | 16 +-
.../flink/FlinkBatchPipelineTranslator.java | 2 +-
.../beam/runners/flink/FlinkPipelineOptions.java | 8 +-
.../flink/FlinkStreamingTransformTranslators.java | 2 +-
.../flink/metrics/FlinkMetricContainerBase.java | 4 +-
.../streaming/io/UnboundedSourceWrapper.java | 2 +-
.../worker/StreamingModeExecutionContext.java | 12 +-
.../dataflow/worker/WindmillTimerInternals.java | 6 +-
.../worker/windmill/state/WindmillTagEncoding.java | 5 +-
.../windmill/state/WindmillTagEncodingV1.java | 3 +-
.../windmill/state/WindmillTagEncodingV2.java | 406 +++++++++++++++
.../processing/ComputationWorkExecutorFactory.java | 14 +-
.../worker/StreamingModeExecutionContextTest.java | 3 +-
.../dataflow/worker/WorkerCustomSourcesTest.java | 6 +-
.../windmill/state/WindmillTagEncodingV2Test.java | 576 +++++++++++++++++++++
sdks/go.mod | 2 +-
sdks/go.sum | 4 +-
.../python/apache_beam/yaml/yaml_transform_test.py | 155 ++++++
sdks/python/tox.ini | 2 +-
settings.gradle.kts | 12 +-
.../content/en/documentation/sdks/yaml-errors.md | 58 +++
.../shortcodes/flink_java_pipeline_options.html | 19 +-
.../shortcodes/flink_python_pipeline_options.html | 19 +-
81 files changed, 1723 insertions(+), 585 deletions(-)
create mode 100644 runners/flink/2.0/build.gradle
copy runners/flink/{1.17 => 2.0}/job-server-container/build.gradle (100%)
copy runners/flink/{1.19 => 2.0}/job-server/build.gradle (95%)
copy runners/flink/{ =>
2.0}/src/main/java/org/apache/beam/runners/flink/FlinkBatchTranslationContext.java
(69%)
copy runners/flink/{ =>
2.0}/src/main/java/org/apache/beam/runners/flink/FlinkExecutionEnvironments.java
(78%)
copy runners/flink/{ =>
2.0}/src/main/java/org/apache/beam/runners/flink/FlinkMiniClusterEntryPoint.java
(95%)
copy runners/flink/{ =>
2.0}/src/main/java/org/apache/beam/runners/flink/FlinkPipelineExecutionEnvironment.java
(69%)
copy runners/flink/{ =>
2.0}/src/main/java/org/apache/beam/runners/flink/FlinkPipelineOptions.java (90%)
copy runners/flink/{ =>
2.0}/src/main/java/org/apache/beam/runners/flink/FlinkPipelineRunner.java (94%)
copy runners/flink/{ =>
2.0}/src/main/java/org/apache/beam/runners/flink/FlinkPipelineTranslator.java
(77%)
copy runners/flink/{ =>
2.0}/src/main/java/org/apache/beam/runners/flink/FlinkStreamingPortablePipelineTranslator.java
(99%)
copy runners/flink/{ =>
2.0}/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java
(99%)
copy runners/flink/{ =>
2.0}/src/main/java/org/apache/beam/runners/flink/FlinkTransformOverrides.java
(90%)
copy runners/flink/{ =>
2.0}/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkDoFnFunction.java
(98%)
copy runners/flink/{ =>
2.0}/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkExecutableStageContextFactory.java
(94%)
copy runners/flink/{ =>
2.0}/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkExecutableStageFunction.java
(99%)
copy runners/flink/{ =>
2.0}/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkExecutableStagePruningFunction.java
(95%)
copy runners/flink/{ =>
2.0}/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkMergingNonShuffleReduceFunction.java
(97%)
copy runners/flink/{ =>
2.0}/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkMultiOutputPruningFunction.java
(96%)
copy runners/flink/{ =>
2.0}/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkPartialReduceFunction.java
(97%)
copy runners/flink/{ =>
2.0}/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkReduceFunction.java
(97%)
copy runners/flink/{ =>
2.0}/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkStatefulDoFnFunction.java
(99%)
copy runners/flink/{ =>
2.0}/src/main/java/org/apache/beam/runners/flink/translation/functions/ImpulseSourceFunction.java
(98%)
copy runners/flink/{ =>
2.0}/src/main/java/org/apache/beam/runners/flink/translation/types/CoderTypeInformation.java
(96%)
copy runners/flink/{ =>
2.0}/src/main/java/org/apache/beam/runners/flink/translation/types/EncodedValueSerializer.java
(89%)
copy runners/flink/{ =>
2.0}/src/main/java/org/apache/beam/runners/flink/translation/types/EncodedValueTypeInformation.java
(94%)
copy runners/flink/{ =>
2.0}/src/main/java/org/apache/beam/runners/flink/translation/types/UnversionedTypeSerializerSnapshot.java
(98%)
copy runners/flink/{1.20 =>
2.0}/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java
(99%)
copy runners/flink/{ =>
2.0}/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/StreamingImpulseSource.java
(86%)
copy runners/flink/{ =>
2.0}/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/TestStreamSource.java
(97%)
copy runners/flink/{ =>
2.0}/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/UnboundedSourceWrapper.java
(95%)
copy runners/flink/{ =>
2.0}/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkBroadcastStateInternals.java
(99%)
copy runners/flink/{ =>
2.0}/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkStateInternals.java
(99%)
copy runners/flink/{ =>
2.0}/src/test/java/org/apache/beam/runners/flink/EncodedValueComparatorTest.java
(96%)
copy runners/flink/{ =>
2.0}/src/test/java/org/apache/beam/runners/flink/FlinkExecutionEnvironmentsTest.java
(91%)
copy runners/flink/{ =>
2.0}/src/test/java/org/apache/beam/runners/flink/FlinkPipelineExecutionEnvironmentTest.java
(90%)
copy runners/flink/{ =>
2.0}/src/test/java/org/apache/beam/runners/flink/FlinkPipelineOptionsTest.java
(95%)
copy runners/flink/{ =>
2.0}/src/test/java/org/apache/beam/runners/flink/FlinkRequiresStableInputTest.java
(97%)
copy runners/flink/{ =>
2.0}/src/test/java/org/apache/beam/runners/flink/FlinkRunnerTest.java (92%)
copy runners/flink/{ =>
2.0}/src/test/java/org/apache/beam/runners/flink/FlinkSavepointTest.java (98%)
copy runners/flink/{ =>
2.0}/src/test/java/org/apache/beam/runners/flink/FlinkSubmissionTest.java (89%)
copy runners/flink/{ =>
2.0}/src/test/java/org/apache/beam/runners/flink/ReadSourceTest.java (81%)
copy runners/flink/{ =>
2.0}/src/test/java/org/apache/beam/runners/flink/adapter/BeamFlinkDataStreamAdapterTest.java
(96%)
copy runners/flink/{ =>
2.0}/src/test/java/org/apache/beam/runners/flink/streaming/BoundedSourceRestoreTest.java
(97%)
copy runners/flink/{ =>
2.0}/src/test/java/org/apache/beam/runners/flink/streaming/FlinkStateInternalsTest.java
(98%)
copy runners/flink/{1.19 =>
2.0}/src/test/java/org/apache/beam/runners/flink/streaming/MemoryStateBackendWrapper.java
(91%)
copy runners/flink/{1.19 =>
2.0}/src/test/java/org/apache/beam/runners/flink/streaming/StreamSources.java
(97%)
copy runners/flink/{ =>
2.0}/src/test/java/org/apache/beam/runners/flink/translation/functions/FlinkDoFnFunctionTest.java
(96%)
copy runners/flink/{ =>
2.0}/src/test/java/org/apache/beam/runners/flink/translation/functions/FlinkExecutableStageFunctionTest.java
(97%)
copy runners/flink/{ =>
2.0}/src/test/java/org/apache/beam/runners/flink/translation/functions/FlinkStatefulDoFnFunctionTest.java
(96%)
copy runners/flink/{ =>
2.0}/src/test/java/org/apache/beam/runners/flink/translation/functions/ImpulseSourceFunctionTest.java
(99%)
copy runners/flink/{ =>
2.0}/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/UnboundedSourceWrapperTest.java
(93%)
copy runners/flink/{ =>
2.0}/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/stableinput/BufferingDoFnRunnerTest.java
(97%)
copy learning/katas/go/common_transforms/aggregation/lesson-info.yaml =>
runners/flink/2.0/src/test/resources/flink-test-config.yaml (85%)
create mode 100644
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillTagEncodingV2.java
create mode 100644
runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillTagEncodingV2Test.java
