This is an automated email from the ASF dual-hosted git repository. bhulette pushed a commit to branch website-revamp in repository https://gitbox.apache.org/repos/asf/beam.git
commit 7d0d220278b65376ab84832421be613e901b1177 Merge: 8b22fb4 53615fe Author: Brian Hulette <[email protected]> AuthorDate: Tue Jan 12 17:53:24 2021 -0800 Merge remote-tracking branch 'origin/master' into website-revamp .github/workflows/build_wheels.yml | 2 +- .github/workflows/cancel.yml | 2 +- .gitignore | 10 + .test-infra/jenkins/README.md | 9 +- .test-infra/jenkins/job_Inventory.groovy | 15 +- .../jenkins/job_LoadTests_Combine_Flink_Go.groovy | 2 +- ...k_Go.groovy => job_LoadTests_Combine_Go.groovy} | 85 +- .../jenkins/job_LoadTests_GBK_Flink_Go.groovy | 19 +- .test-infra/jenkins/job_LoadTests_GBK_Go.groovy | 211 +++++ .test-infra/jenkins/job_LoadTests_ParDo_Go.groovy | 145 ++++ .../job_LoadTests_SideInput_Flink_Go.groovy | 103 +++ .../jenkins/job_LoadTests_SideInput_Go.groovy | 94 +++ .test-infra/jenkins/job_LoadTests_coGBK_Go.groovy | 169 ++++ ...ython36.groovy => job_PostCommit_Python.groovy} | 39 +- .test-infra/jenkins/job_PostCommit_Python37.groovy | 44 - .test-infra/jenkins/job_PostCommit_Python38.groovy | 44 - ...stCommit_Python_ValidatesRunner_Dataflow.groovy | 8 +- ...ommit_Python_ValidatesRunner_Dataflow_V2.groovy | 9 +- .../jenkins/job_Publish_SDK_Image_Snapshots.groovy | 8 +- .../perftests_metrics/SideInput_Load_Tests.json | 2 +- CHANGES.md | 51 +- build.gradle | 9 - buildSrc/build.gradle | 4 +- .../org/apache/beam/gradle/BeamModulePlugin.groovy | 31 +- examples/java/build.gradle | 11 +- .../org/apache/beam/examples/complete/README.md | 5 + .../kafkatopubsub/KafkaPubsubConstants.java | 15 +- .../complete/kafkatopubsub/KafkaToPubsub.java | 235 ++++++ .../beam/examples/complete/kafkatopubsub/README.md | 200 +++++ .../complete/kafkatopubsub/avro/AvroDataClass.java | 63 ++ .../avro/AvroDataClassKafkaAvroDeserializer.java | 27 +- .../complete/kafkatopubsub/avro/package-info.java | 10 +- .../kafka/consumer/SslConsumerFactoryFn.java | 135 ++++ .../kafkatopubsub/kafka/consumer/Utils.java | 165 ++++ .../kafkatopubsub/kafka/consumer/package-info.java | 10 +- .../options/KafkaToPubsubOptions.java | 90 +++ .../kafkatopubsub/options/package-info.java | 10 +- .../complete/kafkatopubsub/package-info.java | 10 +- .../kafkatopubsub/transforms/FormatTransform.java | 128 +++ .../kafkatopubsub/transforms/package-info.java | 10 +- .../examples/complete/game/LeaderBoardTest.java | 2 +- .../complete/kafkatopubsub/KafkaToPubsubTest.java | 90 +++ .../cookbook/CombinePerKeyExamplesTest.java | 13 +- examples/kotlin/build.gradle | 2 +- .../beam/examples/kotlin/MinimalWordCount.kt | 2 +- gradle.properties | 7 +- gradle/wrapper/gradle-wrapper.properties | 2 +- .../beam/learning/katas/util/ContainsKvs.java | 2 +- .../pipeline/src/main/proto/beam_runner_api.proto | 18 + .../src/main/scripts/build_release_candidate.sh | 6 +- .../src/main/scripts/preparation_before_release.sh | 2 + release/src/main/scripts/publish_docker_images.sh | 4 +- .../core/construction/CoderTranslationTest.java | 2 +- .../runners/core/construction/CommonCoderTest.java | 2 +- .../CreatePCollectionViewTranslationTest.java | 2 +- .../DeduplicatedFlattenFactoryTest.java | 2 +- .../construction/DefaultArtifactResolverTest.java | 2 +- .../EmptyFlattenAsCreateFactoryTest.java | 2 +- .../core/construction/EnvironmentsTest.java | 2 +- .../ExecutableStageTranslationTest.java | 2 +- .../construction/ForwardingPTransformTest.java | 2 +- .../construction/GroupByKeyTranslationTest.java | 2 +- .../runners/core/construction/ModelCodersTest.java | 2 +- .../core/construction/MorePipelineTest.java | 2 +- .../construction/PCollectionTranslationTest.java | 2 +- .../core/construction/PTransformMatchersTest.java | 2 +- .../construction/PTransformReplacementsTest.java | 2 +- .../construction/PTransformTranslationTest.java | 2 +- .../core/construction/ParDoTranslationTest.java | 2 +- .../PipelineOptionsTranslationTest.java | 2 +- .../core/construction/PipelineTranslationTest.java | 2 +- .../core/construction/ReadTranslationTest.java | 2 +- .../construction/RehydratedComponentsTest.java | 2 +- .../core/construction/ReplacementOutputsTest.java | 2 +- .../construction/ReshuffleTranslationTest.java | 2 +- .../core/construction/SchemaTranslationTest.java | 2 +- .../core/construction/SdkComponentsTest.java | 2 +- .../SingleInputOutputOverrideFactoryTest.java | 2 +- .../construction/TestStreamTranslationTest.java | 2 +- .../core/construction/TransformInputsTest.java | 2 +- .../core/construction/TriggerTranslationTest.java | 2 +- .../core/construction/UnconsumedReadsTest.java | 2 +- .../core/construction/ValidateRunnerXlangTest.java | 2 +- .../WindowingStrategyTranslationTest.java | 161 +++- .../construction/WriteFilesTranslationTest.java | 2 +- .../construction/graph/ExecutableStageTest.java | 2 +- .../core/construction/graph/FusedPipelineTest.java | 2 +- .../graph/GreedyPipelineFuserTest.java | 2 +- .../construction/graph/GreedyStageFuserTest.java | 2 +- .../graph/ImmutableExecutableStageTest.java | 2 +- .../core/construction/graph/NetworksTest.java | 2 +- .../construction/graph/OutputDeduplicatorTest.java | 2 +- .../construction/graph/ProtoOverridesTest.java | 2 +- .../construction/graph/QueryablePipelineTest.java | 2 +- .../core/metrics/ExecutionStateTracker.java | 20 +- .../runners/core/InMemoryStateInternalsTest.java | 2 +- .../runners/core/InMemoryTimerInternalsTest.java | 2 +- .../core/LateDataDroppingDoFnRunnerTest.java | 2 +- .../beam/runners/core/LateDataUtilsTest.java | 2 +- ...BoundedSplittableProcessElementInvokerTest.java | 2 +- .../beam/runners/core/ReduceFnRunnerTest.java | 2 +- .../beam/runners/core/SideInputHandlerTest.java | 2 +- .../beam/runners/core/SimpleDoFnRunnerTest.java | 2 +- .../SimplePushbackSideInputDoFnRunnerTest.java | 2 +- .../runners/core/SplittableParDoProcessFnTest.java | 2 +- .../beam/runners/core/StateNamespacesTest.java | 2 +- .../beam/runners/core/TimerInternalsTest.java | 2 +- .../beam/runners/core/WindowMatchersTest.java | 2 +- .../beam/runners/core/metrics/CounterCellTest.java | 2 +- .../beam/runners/core/metrics/DirtyStateTest.java | 2 +- .../runners/core/metrics/DistributionCellTest.java | 2 +- .../core/metrics/ExecutionStateSamplerTest.java | 2 +- .../core/metrics/ExecutionStateTrackerTest.java | 2 +- .../beam/runners/core/metrics/GaugeCellTest.java | 2 +- .../core/metrics/MetricsContainerImplTest.java | 2 +- .../core/metrics/MetricsContainerStepMapTest.java | 2 +- .../beam/runners/core/metrics/MetricsMapTest.java | 2 +- .../runners/core/metrics/MetricsPusherTest.java | 2 +- .../core/metrics/MonitoringInfoConstantsTest.java | 2 +- .../core/metrics/SimpleStateRegistryTest.java | 2 +- .../triggers/AfterWatermarkStateMachineTest.java | 2 +- .../core/triggers/FinishedTriggersBitSetTest.java | 2 +- .../core/triggers/FinishedTriggersSetTest.java | 2 +- .../core/triggers/NeverStateMachineTest.java | 2 +- .../core/triggers/TriggerStateMachinesTest.java | 2 +- .../direct/BoundedReadEvaluatorFactoryTest.java | 2 +- .../runners/direct/CloningBundleFactoryTest.java | 2 +- .../beam/runners/direct/CommittedResultTest.java | 2 +- .../CopyOnAccessInMemoryStateInternalsTest.java | 2 +- .../runners/direct/DirectGraphVisitorTest.java | 2 +- .../DirectGroupByKeyOverrideFactoryTest.java | 2 +- .../beam/runners/direct/DirectMetricsTest.java | 2 +- .../runners/direct/DirectRunnerApiSurfaceTest.java | 2 +- .../beam/runners/direct/DirectRunnerTest.java | 2 +- .../runners/direct/DirectTimerInternalsTest.java | 2 +- .../direct/DirectTransformExecutorTest.java | 2 +- ...cycleManagerRemovingTransformEvaluatorTest.java | 2 +- .../runners/direct/DoFnLifecycleManagerTest.java | 2 +- .../beam/runners/direct/EvaluationContextTest.java | 2 +- .../direct/FlattenEvaluatorFactoryTest.java | 2 +- .../direct/GroupByKeyEvaluatorFactoryTest.java | 2 +- .../direct/GroupByKeyOnlyEvaluatorFactoryTest.java | 2 +- .../ImmutabilityCheckingBundleFactoryTest.java | 2 +- .../direct/ImmutableListBundleFactoryTest.java | 2 +- .../direct/ImpulseEvaluatorFactoryTest.java | 2 +- .../direct/KeyedPValueTrackingVisitorTest.java | 2 +- .../beam/runners/direct/MultiStepCombineTest.java | 2 +- .../beam/runners/direct/ParDoEvaluatorTest.java | 2 +- .../runners/direct/SideInputContainerTest.java | 2 +- .../direct/StatefulParDoEvaluatorFactoryTest.java | 2 +- .../runners/direct/StepTransformResultTest.java | 2 +- .../direct/TestStreamEvaluatorFactoryTest.java | 2 +- .../direct/UnboundedReadDeduplicatorTest.java | 2 +- .../direct/UnboundedReadEvaluatorFactoryTest.java | 2 +- .../runners/direct/ViewEvaluatorFactoryTest.java | 2 +- .../direct/WatermarkCallbackExecutorTest.java | 2 +- .../beam/runners/direct/WatermarkManagerTest.java | 2 +- .../runners/direct/WindowEvaluatorFactoryTest.java | 2 +- .../direct/WriteWithShardingFactoryTest.java | 2 +- .../runners/flink/streaming/StreamSources.java | 13 + {sdks/go => runners/flink/1.12}/build.gradle | 38 +- .../flink/1.12/job-server-container/build.gradle | 16 +- .../flink/1.12/job-server}/build.gradle | 34 +- .../streaming/AbstractStreamOperatorCompat.java | 53 ++ .../runners/flink/streaming/StreamSources.java | 14 + .../runners/flink/streaming/StreamSources.java | 13 + .../runners/flink/streaming/StreamSources.java | 13 + runners/flink/flink_runner.gradle | 19 +- .../runners/flink/FlinkExecutionEnvironments.java | 12 + .../flink/FlinkPortableClientEntryPoint.java | 4 +- .../FlinkStreamingPortablePipelineTranslator.java | 27 +- .../flink/FlinkStreamingTransformTranslators.java | 215 +++-- .../flink/FlinkStreamingTranslationContext.java | 24 +- .../wrappers/streaming/DoFnOperator.java | 77 +- .../streaming/KeyedPushedBackElementsHandler.java | 5 +- .../streaming/SingletonKeyedWorkItemCoder.java | 13 +- .../wrappers/streaming/WorkItemKeySelector.java | 13 +- .../streaming/state/FlinkStateInternals.java | 4 +- .../FlinkStreamingPipelineTranslatorTest.java | 109 +++ .../FlinkStreamingTransformTranslatorsTest.java | 7 +- .../runners/flink/FlinkTransformOverridesTest.java | 2 +- .../wrappers/streaming/DedupingOperatorTest.java | 2 +- .../wrappers/streaming/DoFnOperatorTest.java | 315 ++++++-- .../wrappers/streaming/WindowDoFnOperatorTest.java | 5 +- runners/google-cloud-dataflow-java/build.gradle | 2 +- .../beam/runners/dataflow/DataflowRunner.java | 53 +- .../options/DataflowPipelineDebugOptions.java | 10 + .../dataflow/BatchStatefulParDoOverridesTest.java | 2 +- .../runners/dataflow/BatchViewOverridesTest.java | 2 +- .../dataflow/DataflowPTransformMatchersTest.java | 2 +- .../dataflow/DataflowPipelineTranslatorTest.java | 2 +- .../runners/dataflow/DataflowRunnerInfoTest.java | 2 +- .../beam/runners/dataflow/DataflowRunnerTest.java | 56 ++ .../dataflow/PrimitiveParDoSingleFactoryTest.java | 2 +- .../runners/dataflow/TestDataflowRunnerTest.java | 2 +- .../options/DataflowPipelineDebugOptionsTest.java | 2 +- .../options/DataflowProfilingOptionsTest.java | 2 +- .../runners/dataflow/util/PackageUtilTest.java | 2 +- .../beam/runners/dataflow/util/StructsTest.java | 25 +- .../beam/runners/dataflow/worker/ReaderCache.java | 38 +- .../dataflow/worker/StreamingDataflowWorker.java | 61 +- .../worker/fn/data/BeamFnDataGrpcService.java | 14 +- .../logging/DataflowWorkerLoggingInitializer.java | 14 +- .../JulHandlerPrintStreamAdapterFactory.java | 15 +- .../dataflow/harness/test/TestStreamsTest.java | 2 +- .../dataflow/worker/AvroByteReaderFactoryTest.java | 5 +- .../dataflow/worker/AvroByteSinkFactoryTest.java | 3 +- .../dataflow/worker/BatchDataflowWorkerTest.java | 2 +- .../worker/BatchModeExecutionContextTest.java | 2 +- .../dataflow/worker/ConcatReaderFactoryTest.java | 2 +- .../runners/dataflow/worker/ConcatReaderTest.java | 2 +- .../ContextActivationObserverRegistryTest.java | 2 +- ...CreateIsmShardKeyAndSortKeyDoFnFactoryTest.java | 2 +- .../DataflowElementExecutionTrackerTest.java | 2 +- .../worker/DataflowExecutionStateTrackerTest.java | 2 +- .../worker/DataflowSideInputReadCounterTest.java | 2 +- .../worker/DataflowWorkerHarnessHelperTest.java | 2 +- .../dataflow/worker/DefaultParDoFnFactoryTest.java | 2 +- .../dataflow/worker/DeltaCounterCellTest.java | 2 +- .../dataflow/worker/DoFnInstanceManagersTest.java | 2 +- .../dataflow/worker/FakeWindmillServer.java | 14 +- .../runners/dataflow/worker/FilepatternsTest.java | 2 +- .../GroupAlsoByWindowParDoFnFactoryTest.java | 2 +- .../dataflow/worker/InMemoryReaderFactoryTest.java | 3 +- .../IntrinsicMapTaskExecutorFactoryTest.java | 2 +- .../worker/IntrinsicMapTaskExecutorTest.java | 2 +- .../runners/dataflow/worker/IsmFormatTest.java | 2 +- .../dataflow/worker/IsmSideInputReaderTest.java | 2 +- .../dataflow/worker/LogRecordMatcherTest.java | 2 +- .../beam/runners/dataflow/worker/LogSaverTest.java | 2 +- ...tContextActivationObserverRegistrationTest.java | 2 +- .../worker/PairWithConstantKeyDoFnFactoryTest.java | 2 +- .../worker/PartialGroupByKeyParDoFnsTest.java | 2 +- .../runners/dataflow/worker/ReaderCacheTest.java | 6 +- .../runners/dataflow/worker/ReaderFactoryTest.java | 6 +- ...ReifyTimestampAndWindowsParDoFnFactoryTest.java | 2 +- ...essCoderCloudObjectTranslatorRegistrarTest.java | 2 +- .../dataflow/worker/ShuffleReaderFactoryTest.java | 3 +- .../dataflow/worker/ShuffleSinkFactoryTest.java | 3 +- .../runners/dataflow/worker/SimpleParDoFnTest.java | 2 +- .../runners/dataflow/worker/SinkRegistryTest.java | 5 +- .../worker/SourceOperationExecutorFactoryTest.java | 2 +- .../runners/dataflow/worker/StateFetcherTest.java | 2 +- .../worker/StreamingDataflowWorkerTest.java | 10 +- .../worker/StreamingGroupAlsoByWindowFnsTest.java | 2 +- ...reamingGroupAlsoByWindowsReshuffleDoFnTest.java | 2 +- .../worker/StreamingModeExecutionContextTest.java | 5 +- ...eamingPCollectionViewWriterDoFnFactoryTest.java | 2 +- .../worker/StreamingSideInputDoFnRunnerTest.java | 2 +- .../worker/StreamingSideInputFetcherTest.java | 2 +- .../worker/StreamingStepMetricsContainerTest.java | 2 +- .../ToIsmRecordForMultimapDoFnFactoryTest.java | 2 +- .../dataflow/worker/UserParDoFnFactoryTest.java | 2 +- .../dataflow/worker/ValuesDoFnFactoryTest.java | 2 +- .../dataflow/worker/WindmillKeyedWorkItemTest.java | 2 +- .../worker/WindmillStateInternalsTest.java | 2 +- .../dataflow/worker/WindmillStateReaderTest.java | 2 +- .../worker/WindmillTimerInternalsTest.java | 2 +- .../dataflow/worker/WorkItemStatusClientTest.java | 2 +- .../dataflow/worker/WorkerCustomSourcesTest.java | 2 +- .../dataflow/worker/counters/CounterSetTest.java | 2 +- .../fn/control/BeamFnMapTaskExecutorTest.java | 2 +- .../RegisterAndProcessBundleOperationTest.java | 2 +- .../worker/fn/data/BeamFnDataGrpcServiceTest.java | 2 +- .../fn/data/RemoteGrpcPortReadOperationTest.java | 2 +- .../fn/data/RemoteGrpcPortWriteOperationTest.java | 2 +- .../fn/logging/BeamFnLoggingServiceTest.java | 2 +- .../fn/stream/ServerStreamObserverFactoryTest.java | 2 +- .../graph/CloneAmbiguousFlattensFunctionTest.java | 2 +- .../CreateRegisterFnOperationFunctionTest.java | 2 +- .../graph/DeduceNodeLocationsFunctionTest.java | 2 +- ...tFetchAndFilterStreamingSideInputNodesTest.java | 2 +- .../worker/graph/MapTaskToNetworkFunctionTest.java | 2 +- .../dataflow/worker/graph/NetworksTest.java | 2 +- .../RemoveFlattenInstructionsFunctionTest.java | 2 +- .../ReplacePgbkWithPrecombineFunctionTest.java | 2 +- .../DataflowWorkerLoggingInitializerTest.java | 2 +- .../JulHandlerPrintStreamAdapterFactoryTest.java | 16 +- .../worker/profiler/ScopedProfilerTest.java | 2 +- .../dataflow/worker/status/ThreadzServletTest.java | 2 +- .../worker/status/WorkerStatusPagesTest.java | 2 +- .../worker/util/BatchGroupAlsoByWindowFnsTest.java | 2 +- .../BatchGroupAlsoByWindowReshuffleDoFnTest.java | 2 +- .../worker/util/GroupAlsoByWindowProperties.java | 2 +- .../dataflow/worker/util/MemoryMonitorTest.java | 2 +- .../worker/util/ScalableBloomFilterTest.java | 2 +- .../dataflow/worker/util/TimerOrElementTest.java | 2 +- .../worker/BatchingShuffleEntryReaderTest.java | 2 +- .../worker/CachingShuffleBatchReaderTest.java | 2 +- .../util/common/worker/FlattenOperationTest.java | 5 +- .../util/common/worker/GroupingTablesTest.java | 2 +- .../util/common/worker/MapTaskExecutorTest.java | 2 +- .../util/common/worker/OutputReceiverTest.java | 5 +- .../util/common/worker/ParDoOperationTest.java | 2 +- .../util/common/worker/ReadOperationTest.java | 2 +- .../util/common/worker/ShuffleEntryTest.java | 2 +- .../util/common/worker/WriteOperationTest.java | 2 +- .../runners/fnexecution/data/GrpcDataService.java | 6 +- .../runners/fnexecution/ServerFactoryTest.java | 2 +- .../control/DefaultJobBundleFactoryTest.java | 62 +- .../control/FnApiControlClientPoolServiceTest.java | 2 +- .../control/FnApiControlClientTest.java | 2 +- ...gleEnvironmentInstanceJobBundleFactoryTest.java | 2 +- .../fnexecution/data/GrpcDataServiceTest.java | 2 +- .../environment/ProcessEnvironmentFactoryTest.java | 2 +- .../environment/ProcessManagerTest.java | 8 +- .../environment/RemoteEnvironmentTest.java | 2 +- .../logging/GrpcLoggingServiceTest.java | 2 +- .../StaticGrpcProvisionServiceTest.java | 2 +- .../fnexecution/state/GrpcStateServiceTest.java | 2 +- .../translation/PipelineTranslatorUtilsTest.java | 2 +- .../beam/runners/local/StructuralKeyTest.java | 18 +- runners/portability/java/build.gradle | 24 - runners/samza/build.gradle | 17 +- .../beam/runners/samza/SamzaJobInvocation.java | 114 +++ .../beam/runners/samza/SamzaJobServerDriver.java | 60 +- .../beam/runners/samza/SamzaPipelineOptions.java | 22 +- .../samza/SamzaPipelineOptionsValidator.java | 38 +- .../samza/SamzaPortablePipelineOptions.java | 13 + .../runners/samza/SamzaRunnerOverrideConfigs.java | 15 + .../samza/adapter/UnboundedSourceSystem.java | 39 +- .../samza/container/BeamContainerRunner.java | 6 +- .../samza/container/BeamJobCoordinatorRunner.java | 78 ++ ...inerCfgFactory.java => ContainerCfgLoader.java} | 18 +- .../ContainerCfgLoaderFactory.java} | 16 +- .../beam/runners/samza/runtime/BundleManager.java | 349 ++++++++ .../apache/beam/runners/samza/runtime/DoFnOp.java | 277 ++++--- .../runners/samza/runtime/FutureCollector.java | 60 ++ .../beam/runners/samza/runtime/GroupByKeyOp.java | 11 +- .../beam/runners/samza/runtime/KeyedTimerData.java | 1 + .../beam/runners/samza/runtime/OpAdapter.java | 48 +- .../beam/runners/samza/runtime/OpEmitter.java | 5 + .../samza/runtime/OutputManagerFactory.java | 5 + .../samza/runtime/SamzaStoreStateInternals.java | 230 ++++-- .../samza/runtime/SamzaTimerInternalsFactory.java | 217 ++++- .../runners/samza/translation/ConfigBuilder.java | 77 +- .../runners/samza/translation/ConfigContext.java | 8 + .../translation/FlattenPCollectionsTranslator.java | 2 +- .../samza/translation/GroupByKeyTranslator.java | 24 +- .../translation/ParDoBoundMultiTranslator.java | 56 +- .../samza/translation/SamzaPipelineTranslator.java | 21 +- .../translation/SamzaTestStreamSystemFactory.java | 179 +++++ .../translation/SamzaTestStreamTranslator.java | 100 +++ .../translation/SplittableParDoTranslators.java | 6 +- .../samza/translation/TranslationContext.java | 59 +- .../samza/translation/WindowAssignTranslator.java | 4 +- .../beam/runners/samza/util/FutureUtils.java | 50 ++ .../samza/SamzaPipelineOptionsValidatorTest.java | 60 ++ .../samza/adapter/UnboundedSourceSystemTest.java | 28 + .../runners/samza/runtime/BundleManagerTest.java | 474 +++++++++++ .../samza/runtime/FutureCollectorImplTest.java | 92 +++ .../runners/samza/runtime/KeyedTimerDataTest.java | 14 +- .../runtime/SamzaStoreStateInternalsTest.java | 38 +- .../runtime/SamzaTimerInternalsFactoryTest.java | 416 +++++++++- .../samza/translation/ConfigGeneratorTest.java | 86 +- .../samza/translation/TranslationContextTest.java | 94 +++ .../beam/runners/samza/util/FutureUtilsTest.java | 107 +++ .../runners/spark/GlobalWatermarkHolderTest.java | 2 +- .../beam/runners/spark/SparkPipelineStateTest.java | 2 +- .../runners/spark/SparkRunnerDebuggerTest.java | 2 +- .../metrics/sink/SparkMetricsSinkTest.java | 2 +- .../spark/io/ReaderToIteratorAdapterTest.java | 2 +- .../runners/spark/metrics/SparkBeamMetricTest.java | 2 +- .../spark/metrics/SparkMetricsPusherTest.java | 2 +- .../metrics/sink/SparkMetricsSinkTest.java | 2 +- .../metrics/BeamMetricTest.java | 2 +- .../translation/batch/GroupByKeyTest.java | 2 +- .../translation/streaming/CreateStreamTest.java | 2 +- .../ResumeFromCheckpointStreamingTest.java | 2 +- .../streaming/SparkCoGroupByKeyStreamingTest.java | 2 +- .../streaming/StreamingSourceMetricsTest.java | 2 +- .../streaming/TrackStreamingSourcesTest.java | 2 +- sdks/go/build.gradle | 2 +- sdks/go/container/build.gradle | 2 + sdks/go/pkg/beam/core/graph/coder/double_test.go | 53 ++ sdks/go/pkg/beam/core/graph/coder/map_test.go | 11 +- sdks/go/pkg/beam/core/graph/coder/row.go | 337 ++------ sdks/go/pkg/beam/core/graph/coder/row_decoder.go | 308 +++++++ sdks/go/pkg/beam/core/graph/coder/row_encoder.go | 271 +++++++ sdks/go/pkg/beam/core/graph/coder/row_test.go | 488 +++++++++--- .../pkg/beam/core/graph/coder/testutil/testutil.go | 154 ++++ .../core/graph/coder/testutil/testutil_test.go | 201 +++++ sdks/go/pkg/beam/core/metrics/metrics.go | 39 + sdks/go/pkg/beam/core/runtime/graphx/xlang.go | 9 +- sdks/go/pkg/beam/core/runtime/metricsx/metricsx.go | 38 +- sdks/go/pkg/beam/pipeline.go | 1 + sdks/go/pkg/beam/runners/dataflow/dataflow.go | 5 +- .../beam/runners/dataflow/dataflowlib/execute.go | 57 +- .../pkg/beam/runners/dataflow/dataflowlib/job.go | 8 + .../beam/runners/dataflow/dataflowlib/metrics.go | 124 +++ .../runners/dataflow/dataflowlib/metrics_test.go | 133 ++++ .../beam/runners/universal/runnerlib/compile.go | 13 +- .../beam/runners/universal/runnerlib/execute.go | 21 +- sdks/go/pkg/beam/runners/universal/universal.go | 1 + sdks/go/pkg/beam/testing/ptest/ptest.go | 21 + sdks/go/test/build.gradle | 94 ++- sdks/go/test/integration/{ => driver}/driver.go | 0 sdks/go/test/integration/integration.go | 147 ++++ sdks/go/test/integration/primitives/cogbk_test.go | 16 +- .../go/test/integration/primitives/flatten_test.go | 11 +- sdks/go/test/integration/primitives/pardo_test.go | 16 +- .../test/integration/synthetic/synthetic_test.go | 75 ++ .../test/integration/wordcount/wordcount_test.go | 6 + sdks/go/test/integration/xlang/xlang_test.go | 260 ++++++ sdks/go/test/load/build.gradle | 1 + sdks/go/test/load/sideinput/sideinput.go | 100 +++ sdks/go/test/run_integration_tests.sh | 2 +- ...ation_tests.sh => run_validatesrunner_tests.sh} | 220 +++-- .../src/main/resources/beam/checkstyle.xml | 17 + .../src/main/resources/beam/suppressions.xml | 4 + sdks/java/container/build.gradle | 4 +- sdks/java/container/common.gradle | 12 +- sdks/java/core/build.gradle | 5 +- .../main/java/org/apache/beam/sdk/io/AvroIO.java | 38 +- .../apache/beam/sdk/io/AvroSchemaIOProvider.java | 4 +- .../beam/sdk/io/GenericRecordWriteConverter.java | 67 -- .../src/main/java/org/apache/beam/sdk/io/Read.java | 97 ++- .../sdk/schemas/FieldValueTypeInformation.java | 53 +- .../org/apache/beam/sdk/schemas/RowMessages.java | 132 +++ .../beam/sdk/schemas/transforms/Convert.java | 5 + .../apache/beam/sdk/schemas/utils/AvroUtils.java | 4 +- .../beam/sdk/schemas/utils/ConvertHelpers.java | 14 +- .../apache/beam/sdk/testing/CoderProperties.java | 2 +- .../apache/beam/sdk/testing/CombineFnTester.java | 2 +- .../java/org/apache/beam/sdk/testing/PAssert.java | 2 +- .../apache/beam/sdk/testing/SourceTestUtils.java | 2 +- .../org/apache/beam/sdk/testing/TestPipeline.java | 45 ++ .../apache/beam/sdk/testing/WindowFnTestUtils.java | 2 +- .../beam/sdk/transforms/ApproximateUnique.java | 20 +- .../beam/sdk/transforms/windowing/Trigger.java | 3 +- .../resources/org/apache/beam/sdk/sdk.properties | 5 +- .../java/org/apache/beam/sdk/PipelineTest.java | 2 +- .../org/apache/beam/sdk/coders/AvroCoderTest.java | 2 +- .../beam/sdk/coders/BigDecimalCoderTest.java | 2 +- .../beam/sdk/coders/BigEndianIntegerCoderTest.java | 2 +- .../beam/sdk/coders/BigEndianLongCoderTest.java | 2 +- .../beam/sdk/coders/BigIntegerCoderTest.java | 2 +- .../apache/beam/sdk/coders/BitSetCoderTest.java | 2 +- .../apache/beam/sdk/coders/ByteArrayCoderTest.java | 2 +- .../org/apache/beam/sdk/coders/ByteCoderTest.java | 2 +- .../java/org/apache/beam/sdk/coders/CoderTest.java | 2 +- .../beam/sdk/coders/CollectionCoderTest.java | 2 +- .../apache/beam/sdk/coders/DefaultCoderTest.java | 2 +- .../apache/beam/sdk/coders/DelegateCoderTest.java | 2 +- .../apache/beam/sdk/coders/DoubleCoderTest.java | 2 +- .../apache/beam/sdk/coders/DurationCoderTest.java | 2 +- .../org/apache/beam/sdk/coders/FloatCoderTest.java | 2 +- .../apache/beam/sdk/coders/InstantCoderTest.java | 2 +- .../apache/beam/sdk/coders/IterableCoderTest.java | 2 +- .../org/apache/beam/sdk/coders/KvCoderTest.java | 2 +- .../org/apache/beam/sdk/coders/ListCoderTest.java | 2 +- .../org/apache/beam/sdk/coders/MapCoderTest.java | 2 +- .../apache/beam/sdk/coders/NullableCoderTest.java | 2 +- .../sdk/coders/PCollectionCustomCoderTest.java | 2 +- .../beam/sdk/coders/SerializableCoderTest.java | 2 +- .../org/apache/beam/sdk/coders/SetCoderTest.java | 2 +- .../beam/sdk/coders/StringDelegateCoderTest.java | 2 +- .../beam/sdk/coders/StringUtf8CoderTest.java | 2 +- .../beam/sdk/coders/StructuredCoderTest.java | 11 +- .../beam/sdk/coders/TextualIntegerCoderTest.java | 2 +- .../apache/beam/sdk/coders/VarIntCoderTest.java | 2 +- .../apache/beam/sdk/coders/VarLongCoderTest.java | 2 +- .../org/apache/beam/sdk/coders/VoidCoderTest.java | 2 +- .../java/org/apache/beam/sdk/io/AvroIOTest.java | 2 +- .../org/apache/beam/sdk/io/AvroSourceTest.java | 2 +- .../sdk/io/BoundedReadFromUnboundedSourceTest.java | 2 +- .../apache/beam/sdk/io/CompressedSourceTest.java | 2 +- .../org/apache/beam/sdk/io/CountingSourceTest.java | 2 +- .../org/apache/beam/sdk/io/FileBasedSinkTest.java | 2 +- .../apache/beam/sdk/io/FileBasedSourceTest.java | 2 +- .../org/apache/beam/sdk/io/FileSystemsTest.java | 2 +- .../beam/sdk/io/LocalFileSystemRegistrarTest.java | 2 +- .../apache/beam/sdk/io/LocalFileSystemTest.java | 2 +- .../test/java/org/apache/beam/sdk/io/ReadTest.java | 130 +++ .../org/apache/beam/sdk/io/TextIOReadTest.java | 2 +- .../org/apache/beam/sdk/io/TextIOWriteTest.java | 2 +- .../org/apache/beam/sdk/io/WriteFilesTest.java | 2 +- .../io/range/ByteKeyRangeEstimateFractionTest.java | 2 +- .../io/range/ByteKeyRangeInterpolateKeyTest.java | 2 +- .../apache/beam/sdk/io/range/ByteKeyRangeTest.java | 2 +- .../org/apache/beam/sdk/io/range/ByteKeyTest.java | 2 +- .../org/apache/beam/sdk/metrics/MetricsTest.java | 2 +- .../sdk/options/PipelineOptionsFactoryTest.java | 2 +- .../sdk/options/ProxyInvocationHandlerTest.java | 2 +- .../beam/sdk/runners/PipelineRunnerTest.java | 2 +- .../beam/sdk/runners/TransformHierarchyTest.java | 2 +- .../apache/beam/sdk/runners/TransformTreeTest.java | 2 +- .../beam/sdk/schemas/AutoValueSchemaTest.java | 4 +- .../beam/sdk/schemas/JavaBeanSchemaTest.java | 38 +- .../beam/sdk/schemas/JavaFieldSchemaTest.java | 6 +- .../apache/beam/sdk/schemas/SchemaCoderTest.java | 2 +- .../sdk/schemas/transforms/CastValidatorTest.java | 2 +- .../beam/sdk/schemas/transforms/CoGroupTest.java | 2 +- .../beam/sdk/schemas/transforms/ConvertTest.java | 15 + .../beam/sdk/schemas/transforms/GroupTest.java | 2 +- .../beam/sdk/schemas/utils/JavaBeanUtilsTest.java | 26 +- .../beam/sdk/schemas/utils/POJOUtilsTest.java | 6 +- .../beam/sdk/schemas/utils/SchemaZipFoldTest.java | 2 +- .../beam/sdk/schemas/utils/TestJavaBeans.java | 17 + .../apache/beam/sdk/state/StateContextsTest.java | 2 +- .../beam/sdk/testing/CoderPropertiesTest.java | 2 +- .../beam/sdk/testing/CombineFnTesterTest.java | 2 +- .../org/apache/beam/sdk/testing/PAssertTest.java | 2 +- .../beam/sdk/testing/PaneExtractorsTest.java | 2 +- .../beam/sdk/testing/SerializableMatchersTest.java | 2 +- .../apache/beam/sdk/testing/StaticWindowsTest.java | 2 +- .../apache/beam/sdk/testing/TestPipelineTest.java | 2 +- .../beam/sdk/testing/WindowSupplierTest.java | 2 +- .../apache/beam/sdk/transforms/CombineFnsTest.java | 2 +- .../apache/beam/sdk/transforms/CombineTest.java | 2 +- .../org/apache/beam/sdk/transforms/CreateTest.java | 2 +- .../apache/beam/sdk/transforms/DistinctTest.java | 2 +- .../org/apache/beam/sdk/transforms/DoFnTest.java | 2 +- .../apache/beam/sdk/transforms/DoFnTesterTest.java | 2 +- .../apache/beam/sdk/transforms/FlattenTest.java | 2 +- .../apache/beam/sdk/transforms/GroupByKeyTest.java | 2 +- .../apache/beam/sdk/transforms/PTransformTest.java | 2 +- .../org/apache/beam/sdk/transforms/ParDoTest.java | 2 +- .../org/apache/beam/sdk/transforms/ReifyTest.java | 2 +- .../beam/sdk/transforms/ReifyTimestampsTest.java | 2 +- .../apache/beam/sdk/transforms/ReshuffleTest.java | 2 +- .../beam/sdk/transforms/SimpleFunctionTest.java | 2 +- .../beam/sdk/transforms/WithFailuresTest.java | 2 +- .../display/DisplayDataEvaluatorTest.java | 2 +- .../display/DisplayDataMatchersTest.java | 2 +- .../sdk/transforms/display/DisplayDataTest.java | 2 +- .../beam/sdk/transforms/join/CoGbkResultTest.java | 2 +- .../beam/sdk/transforms/join/CoGroupByKeyTest.java | 2 +- .../beam/sdk/transforms/join/UnionCoderTest.java | 2 +- .../sdk/transforms/reflect/DoFnInvokersTest.java | 2 +- .../sdk/transforms/reflect/DoFnSignaturesTest.java | 2 +- .../transforms/reflect/OnTimerInvokersTest.java | 2 +- .../sdk/transforms/windowing/FixedWindowsTest.java | 2 +- .../transforms/windowing/IntervalWindowTest.java | 2 +- .../sdk/transforms/windowing/RepeatedlyTest.java | 2 +- .../sdk/transforms/windowing/SessionsTest.java | 2 +- .../transforms/windowing/SlidingWindowsTest.java | 2 +- .../beam/sdk/transforms/windowing/WindowTest.java | 2 +- .../org/apache/beam/sdk/util/ApiSurfaceTest.java | 2 +- .../BufferedElementCountingOutputStreamTest.java | 2 +- .../apache/beam/sdk/util/FluentBackoffTest.java | 2 +- .../org/apache/beam/sdk/util/HistogramTest.java | 2 +- .../org/apache/beam/sdk/util/MoreFuturesTest.java | 2 +- .../org/apache/beam/sdk/util/ReleaseInfoTest.java | 2 +- .../beam/sdk/util/UserCodeExceptionTest.java | 2 +- .../java/org/apache/beam/sdk/util/VarIntTest.java | 2 +- .../apache/beam/sdk/util/WindowedValueTest.java | 2 +- .../org/apache/beam/sdk/util/ZipFilesTest.java | 2 +- .../beam/sdk/util/common/ReflectHelpersTest.java | 2 +- .../java/org/apache/beam/sdk/values/KVTest.java | 2 +- .../beam/sdk/values/PCollectionListTest.java | 2 +- .../beam/sdk/values/PCollectionTupleTest.java | 2 +- .../org/apache/beam/sdk/values/TupleTagTest.java | 2 +- .../beam/sdk/values/TypeDescriptorsTest.java | 2 +- .../apache/beam/sdk/values/TypedPValueTest.java | 2 +- .../beam/sdk/expansion/service/ExternalTest.java | 2 +- .../sdk/extensions/gcp/util/GceMetadataUtil.java | 4 +- .../gcp/storage/GcsFileSystemRegistrarTest.java | 2 +- .../extensions/gcp/storage/GcsFileSystemTest.java | 2 +- .../beam/sdk/extensions/gcp/util/GcsUtilTest.java | 2 +- ...LatencyRecordingHttpRequestInitializerTest.java | 2 +- .../gcp/util/RetryHttpRequestInitializerTest.java | 2 +- .../sdk/extensions/gcp/util/gcsfs/GcsPathTest.java | 2 +- .../extensions/protobuf/ProtoMessageSchema.java | 68 +- .../extensions/protobuf/ByteStringCoderTest.java | 2 +- .../sdk/extensions/protobuf/ProtobufUtilTest.java | 2 +- .../ExternalSchemaIOTransformRegistrarTest.java | 4 +- .../extensions/sketching/TDigestQuantilesTest.java | 2 +- .../extensions/sorter/ExternalSorterBenchmark.java | 6 +- .../beam/sdk/extensions/sorter/SortValuesTest.java | 2 +- sdks/java/extensions/sql/build.gradle | 15 +- .../datacatalog/DataCatalogTableProvider.java | 61 +- .../sql/meta/provider/datacatalog/SchemaUtils.java | 93 +++ .../meta/provider/datacatalog/SchemaUtilsTest.java | 157 ++++ .../sdk/extensions/sql/jdbc/BeamSqlLineIT.java | 2 +- .../sdk/extensions/sql/jdbc/BeamSqlLineTest.java | 2 +- .../sdk/extensions/sql/impl/ScalarFnReflector.java | 73 ++ .../extensions/sql/impl/ScalarFunctionImpl.java | 47 +- .../sql/meta/provider/bigtable/BigtableFilter.java | 136 ++++ .../sql/meta/provider/bigtable/BigtableTable.java | 47 +- .../meta/provider/kafka/BeamKafkaThriftTable.java | 138 ++++ .../meta/provider/kafka/KafkaTableProvider.java | 56 +- .../sql/BeamSqlDslSqlStdOperatorsTest.java | 2 +- .../sdk/extensions/sql/impl/JdbcDriverTest.java | 2 +- .../extensions/sql/impl/ScalarFnReflectorTest.java | 141 ++++ .../provider/bigtable/BigtableClientWrapper.java | 115 +++ .../meta/provider/bigtable/BigtableFilterTest.java | 114 +++ .../BigtableTableCreationFailuresTest.java | 2 +- .../provider/bigtable/BigtableTableFlatTest.java | 132 +-- .../meta/provider/bigtable/BigtableTableIT.java | 200 +++++ .../meta/provider/bigtable/BigtableTableTest.java | 94 --- .../provider/bigtable/BigtableTableTestUtils.java | 237 ++++++ .../bigtable/BigtableTableWithRowsTest.java | 120 ++- .../provider/kafka/BeamKafkaTableProtoTest.java | 2 +- ...rotoTest.java => BeamKafkaTableThriftTest.java} | 53 +- .../meta/provider/kafka/KafkaTableProviderIT.java | 214 ++++- .../provider/kafka/KafkaTableProviderProtoIT.java | 53 -- .../provider/kafka/KafkaTableProviderTest.java | 55 +- .../provider/kafka/thrift/ItThriftMessage.java | 612 ++++++++++++++ .../provider/kafka/thrift/SimpleThriftMessage.java | 509 ++++++++++++ .../provider/kafka/thrift/TestThriftMessage.java | 881 +++++++++++++++++++++ .../sql/meta/provider/pubsub/PubsubAvroIT.java | 102 --- .../sql/meta/provider/pubsub/PubsubJsonIT.java | 76 -- .../provider/pubsub/PubsubTableProviderIT.java | 273 +++++-- .../meta/provider/text/TextTableProviderTest.java | 2 +- .../sql/meta/store/InMemoryMetaStoreTest.java | 2 +- .../sql/src/test/thrift/kafka/messages.thrift | 36 +- sdks/{go => java/extensions/sql/udf}/build.gradle | 27 +- .../beam/sdk/extensions/sql/udf/AggregateFn.java | 71 ++ .../beam/sdk/extensions/sql/udf/ScalarFn.java | 53 ++ .../beam/sdk/extensions/sql/udf/UdfProvider.java} | 23 +- .../beam/sdk/extensions/sql/udf/package-info.java | 16 +- .../sdk/extensions/sql/zetasql/SqlAnalyzer.java | 6 + .../extensions/sql/zetasql/ZetaSQLPlannerImpl.java | 5 +- .../sql/zetasql/translation/ConversionContext.java | 2 +- .../zetasql/translation/ExpressionConverter.java | 11 +- .../sql/zetasql/translation/SqlOperators.java | 76 +- .../translation/UserFunctionDefinitions.java | 30 +- .../translation/ZetaSqlScalarFunctionImpl.java | 86 ++ .../zetasketch/ApproximateCountDistinct.java | 288 +++++++ .../zetasketch/ApproximateCountDistinctTest.java | 342 ++++++++ .../sdk/fn/data/BeamFnDataInboundObserver.java | 35 +- .../beam/sdk/fn/data/DecodingFnDataReceiver.java | 46 ++ .../org/apache/beam/sdk/fn/IdGeneratorsTest.java | 2 +- .../sdk/fn/channel/SocketAddressFactoryTest.java | 2 +- .../sdk/fn/data/BeamFnDataGrpcMultiplexerTest.java | 2 +- ...DataSizeBasedBufferingOutboundObserverTest.java | 2 +- .../CompletableFutureInboundDataClientTest.java | 2 +- .../beam/sdk/fn/data/RemoteGrpcPortReadTest.java | 2 +- .../beam/sdk/fn/data/RemoteGrpcPortWriteTest.java | 2 +- .../fn/splittabledofn/RestrictionTrackersTest.java | 2 +- .../beam/sdk/fn/stream/AdvancingPhaserTest.java | 2 +- .../apache/beam/sdk/fn/stream/DataStreamsTest.java | 2 +- .../sdk/fn/stream/OutboundObserverFactoryTest.java | 2 +- .../apache/beam/sdk/fn/test/TestStreamsTest.java | 2 +- .../fn/harness/control/ProcessBundleHandler.java | 87 +- .../beam/fn/harness/data/BeamFnDataClient.java | 14 +- .../beam/fn/harness/data/BeamFnDataGrpcClient.java | 10 +- .../fn/harness/data/QueueingBeamFnDataClient.java | 11 +- .../fn/harness/logging/BeamFnLoggingClient.java | 15 + .../beam/fn/harness/AssignWindowsRunnerTest.java | 2 +- .../beam/fn/harness/BeamFnDataReadRunnerTest.java | 2 +- .../beam/fn/harness/BeamFnDataWriteRunnerTest.java | 2 +- .../apache/beam/fn/harness/CombineRunnersTest.java | 2 +- .../apache/beam/fn/harness/FlattenRunnerTest.java | 2 +- .../beam/fn/harness/FnApiDoFnRunnerTest.java | 2 +- .../apache/beam/fn/harness/MapFnRunnersTest.java | 2 +- .../fn/harness/PrecombineGroupingTableTest.java | 2 +- .../beam/fn/harness/WindowMergingFnRunnerTest.java | 2 +- .../harness/control/BeamFnControlClientTest.java | 2 +- .../harness/control/BundleSplitListenerTest.java | 2 +- .../harness/control/ProcessBundleHandlerTest.java | 2 +- .../fn/harness/data/BeamFnDataGrpcClientTest.java | 2 +- .../data/BeamFnDataInboundObserverTest.java | 13 +- .../data/PCollectionConsumerRegistryTest.java | 2 +- .../harness/data/QueueingBeamFnDataClientTest.java | 2 +- .../harness/logging/BeamFnLoggingClientTest.java | 2 +- .../state/BeamFnStateGrpcClientCacheTest.java | 2 +- .../stream/HarnessStreamObserverFactoriesTest.java | 2 +- .../beam/sdk/io/aws/dynamodb/DynamoDBIO.java | 40 +- .../beam/sdk/io/aws/coders/AwsCodersTest.java | 2 +- .../beam/sdk/io/aws/dynamodb/DynamoDBIOTest.java | 68 +- .../sdk/io/aws/dynamodb/DynamoDBIOTestHelper.java | 2 +- .../beam/sdk/io/aws/options/AwsModuleTest.java | 2 +- .../beam/sdk/io/aws/s3/S3FileSystemTest.java | 2 +- .../sdk/io/aws/sns/PublishResultCodersTest.java | 2 +- .../aws2/dynamodb/BasicDynamoDbClientProvider.java | 32 +- .../beam/sdk/io/aws2/dynamodb/DynamoDBIO.java | 40 +- .../sdk/io/aws2/options/AwsSerializableUtils.java | 50 ++ .../io/aws2/sns/BasicSnsAsyncClientProvider.java | 32 +- .../sdk/io/aws2/sns/BasicSnsClientProvider.java | 34 +- .../sdk/io/aws2/sqs/BasicSqsClientProvider.java | 34 +- .../dynamodb/BasicDynamoDbClientProviderTest.java | 51 ++ .../beam/sdk/io/aws2/dynamodb/DynamoDBIOTest.java | 85 +- .../beam/sdk/io/aws2/options/AwsModuleTest.java | 2 +- .../io/aws2/options/AwsSerializableUtilsTest.java | 70 ++ .../aws2/sns/BasicSnsAsyncClientProviderTest.java | 51 ++ .../io/aws2/sns/BasicSnsClientProviderTest.java | 51 ++ .../io/aws2/sqs/BasicSqsClientProviderTest.java | 51 ++ .../blobstore/AzureBlobStoreFileSystemTest.java | 2 +- .../beam/sdk/io/cassandra/CassandraIOTest.java | 2 +- .../io/contextualtextio/ContextualTextIOTest.java | 2 +- .../elasticsearch/ElasticsearchIOTestCommon.java | 2 +- .../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 7 +- .../gcp/bigquery/BigQueryStorageStreamSource.java | 6 + .../apache/beam/sdk/io/gcp/bigtable/RowUtils.java | 10 + .../apache/beam/sdk/io/gcp/healthcare/FhirIO.java | 232 ++++++ .../sdk/io/gcp/healthcare/HealthcareApiClient.java | 17 + .../io/gcp/healthcare/HttpHealthcareApiClient.java | 174 ++++ .../sdk/io/gcp/healthcare/JsonArrayCoder.java} | 34 +- .../beam/sdk/io/gcp/pubsub/RowToPubsubMessage.java | 28 +- .../apache/beam/sdk/io/gcp/pubsub/TestPubsub.java | 17 +- .../beam/sdk/io/gcp/pubsub/TestPubsubSignal.java | 150 ++-- .../sdk/io/gcp/pubsublite/CloudPubsubChecks.java | 21 +- ...fsetFinalizer.java => InitialOffsetReader.java} | 12 +- .../io/gcp/pubsublite/InitialOffsetReaderImpl.java | 54 ++ .../gcp/pubsublite/LimitingTopicBacklogReader.java | 81 ++ ...ffsetFinalizer.java => OffsetByteProgress.java} | 17 +- .../io/gcp/pubsublite/OffsetByteRangeTracker.java | 172 ++++ .../io/gcp/pubsublite/OffsetCheckpointMark.java | 78 -- .../pubsublite/PerSubscriptionPartitionSdf.java | 110 +++ .../beam/sdk/io/gcp/pubsublite/PublisherCache.java | 10 +- .../sdk/io/gcp/pubsublite/PublisherOptions.java | 47 +- .../sdk/io/gcp/pubsublite/PublisherOrError.java | 6 +- .../beam/sdk/io/gcp/pubsublite/Publishers.java | 56 ++ .../beam/sdk/io/gcp/pubsublite/PubsubLiteIO.java | 14 +- .../beam/sdk/io/gcp/pubsublite/PubsubLiteSink.java | 18 +- .../gcp/pubsublite/PubsubLiteUnboundedReader.java | 333 -------- .../gcp/pubsublite/PubsubLiteUnboundedSource.java | 140 ---- ...zer.java => SerializableSubscriberFactory.java} | 17 +- .../io/gcp/pubsublite/SerializableSupplier.java | 5 +- .../sdk/io/gcp/pubsublite/SubscribeTransform.java | 124 +++ .../sdk/io/gcp/pubsublite/SubscriberOptions.java | 157 ++-- ...etFinalizer.java => SubscriptionPartition.java} | 19 +- ...dCoder.java => SubscriptionPartitionCoder.java} | 36 +- ...er.java => SubscriptionPartitionProcessor.java} | 16 +- ... => SubscriptionPartitionProcessorFactory.java} | 15 +- .../SubscriptionPartitionProcessorImpl.java | 152 ++++ .../sdk/io/gcp/pubsublite/TopicBacklogReader.java | 34 +- .../io/gcp/pubsublite/TopicBacklogReaderImpl.java | 80 +- .../gcp/pubsublite/TopicBacklogReaderSettings.java | 50 +- .../gcp/pubsublite/TranslatingPullSubscriber.java | 48 -- .../beam/sdk/io/gcp/pubsublite/UuidCoder.java | 7 +- .../gcp/pubsublite/UuidDeduplicationOptions.java | 15 +- .../beam/sdk/io/gcp/spanner/SpannerAccessor.java | 22 +- .../io/gcp/testing/BigtableEmulatorWrapper.java | 75 -- .../beam/sdk/io/gcp/testing/BigtableTestUtils.java | 154 ---- .../beam/sdk/io/gcp/testing/BigtableUtils.java | 52 +- .../cloud/spanner/FakeBatchTransactionId.java | 1 + .../apache/beam/sdk/io/gcp/GcpApiSurfaceTest.java | 3 +- .../sdk/io/gcp/bigquery/BigQueryIOReadTest.java | 2 +- .../io/gcp/bigquery/BigQueryIOStorageReadTest.java | 13 +- .../sdk/io/gcp/bigquery/BigQueryIOWriteTest.java | 2 +- .../io/gcp/bigquery/BigQueryServicesImplTest.java | 2 +- .../sdk/io/gcp/bigquery/BigQueryUtilsTest.java | 2 +- .../sdk/io/gcp/bigquery/TableRowJsonCoderTest.java | 2 +- .../bigtable/BeamRowToBigtableMutationTest.java | 14 +- .../sdk/io/gcp/bigtable/BigtableConfigTest.java | 2 +- .../beam/sdk/io/gcp/bigtable/BigtableIOTest.java | 2 +- .../gcp/bigtable/BigtableRowToBeamRowFlatTest.java | 14 +- .../io/gcp/bigtable/BigtableRowToBeamRowTest.java | 17 +- .../{TestUtils.java => BigtableTestUtils.java} | 50 +- .../beam/sdk/io/gcp/bigtable/BigtableWriteIT.java | 2 +- .../sdk/io/gcp/bigtable/CellValueParserTest.java | 13 +- .../io/gcp/datastore/AdaptiveThrottlerTest.java | 2 +- .../beam/sdk/io/gcp/datastore/DatastoreV1Test.java | 2 +- .../gcp/datastore/EntityToRowRowToEntityTest.java | 6 +- .../beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java | 156 ++++ .../beam/sdk/io/gcp/healthcare/FhirIOTest.java | 21 + .../beam/sdk/io/gcp/pubsub/PubsubIOTest.java | 2 +- .../pubsub/PubsubMessagePayloadOnlyCoderTest.java | 2 +- ...MessageWithAttributesAndMessageIdCoderTest.java | 2 +- .../PubsubMessageWithAttributesCoderTest.java | 2 +- .../PubsubMessageWithMessageIdCoderTest.java | 2 +- .../io/gcp/pubsub/PubsubUnboundedSourceTest.java | 2 +- .../gcp/pubsublite/OffsetByteRangeTrackerTest.java | 158 ++++ .../gcp/pubsublite/OffsetCheckpointMarkTest.java | 81 -- .../PerSubscriptionPartitionSdfTest.java | 189 +++++ .../sdk/io/gcp/pubsublite/PubsubLiteSinkTest.java | 47 +- .../pubsublite/PubsubLiteUnboundedReaderTest.java | 330 -------- .../SubscriptionPartitionProcessorImplTest.java | 227 ++++++ .../gcp/pubsublite/TopicBacklogReaderImplTest.java | 140 +--- .../pubsublite/UuidDeduplicationTransformTest.java | 13 +- .../io/gcp/spanner/MutationSizeEstimatorTest.java | 2 +- .../sdk/io/gcp/spanner/ReadSpannerSchemaTest.java | 2 +- .../sdk/io/gcp/spanner/SpannerIOWriteTest.java | 2 +- .../beam/sdk/io/gcp/spanner/SpannerWriteIT.java | 2 +- .../beam/sdk/io/gcp/spanner/StructUtilsTest.java | 2 +- .../sdk/io/hadoop/SerializableConfiguration.java | 11 + .../io/hadoop/SerializableConfigurationTest.java | 9 + .../beam/sdk/io/hadoop/WritableCoderTest.java | 2 +- .../beam/sdk/io/hdfs/HadoopFileSystemOptions.java | 9 +- .../sdk/io/hdfs/HadoopFileSystemModuleTest.java | 2 +- .../hdfs/HadoopFileSystemOptionsRegistrarTest.java | 2 +- .../sdk/io/hdfs/HadoopFileSystemOptionsTest.java | 22 +- .../beam/sdk/io/hdfs/HadoopFileSystemTest.java | 2 +- .../beam/sdk/io/hadoop/format/HadoopFormatIO.java | 85 +- .../hadoop/format/HadoopFormatIOCassandraTest.java | 6 +- .../io/hadoop/format/HadoopFormatIOReadTest.java | 162 +++- .../org/apache/beam/sdk/io/hbase/HBaseIOTest.java | 2 +- .../beam/sdk/io/hcatalog/HCatalogIOTest.java | 2 +- .../org/apache/beam/sdk/io/jdbc/JdbcIOTest.java | 2 +- .../java/org/apache/beam/sdk/io/jms/JmsIOTest.java | 2 +- .../CustomTimestampPolicyWithLimitedDelayTest.java | 2 +- .../org/apache/beam/sdk/io/kafka/KafkaIOTest.java | 8 +- .../apache/beam/sdk/io/kinesis/KinesisIOIT.java | 2 +- .../java/org/apache/beam/sdk/io/kudu/KuduIOIT.java | 4 +- .../sdk/io/parquet/GenericRecordReadConverter.java | 64 -- .../org/apache/beam/sdk/io/parquet/ParquetIO.java | 472 +++++++++-- .../sdk/io/parquet/ParquetSchemaIOProvider.java | 11 +- .../sdk/io/parquet/GenericRecordToRowTest.java | 79 -- .../apache/beam/sdk/io/parquet/ParquetIOTest.java | 132 ++- .../apache/beam/sdk/io/snowflake/KeyPairUtils.java | 4 +- .../io/snowflake/crosslanguage/ReadBuilder.java | 4 +- .../services/SnowflakeBatchServiceImpl.java | 4 +- .../test/FakeSnowflakeStreamingServiceImpl.java | 4 +- .../beam/sdk/io/snowflake/test/TestUtils.java | 6 +- .../beam/sdk/io/solr/JavaBinCodecCoderTest.java | 2 +- .../org/apache/beam/sdk/io/solr/SolrIOTest.java | 4 +- sdks/java/io/thrift/build.gradle | 10 +- .../org/apache/beam/sdk/io/thrift/ThriftCoder.java | 9 +- .../apache/beam/sdk/io/thrift/ThriftSchema.java | 408 ++++++++++ .../apache/beam/sdk/io/thrift/TestThriftEnum.java} | 45 +- .../beam/sdk/io/thrift/TestThriftInnerStruct.java | 526 ++++++++++++ .../beam/sdk/io/thrift/TestThriftStruct.java | 760 ++++++++++++++++-- .../apache/beam/sdk/io/thrift/TestThriftUnion.java | 401 ++++++++++ .../apache/beam/sdk/io/thrift/ThriftIOTest.java | 38 +- .../beam/sdk/io/thrift/ThriftSchemaTest.java | 251 ++++++ .../src/test/resources/thrift/thrift_test.thrift | 35 +- .../apache/beam/sdk/io/tika/ParseResultTest.java | 2 +- .../org/apache/beam/sdk/io/tika/TikaIOTest.java | 2 +- .../java/org/apache/beam/sdk/io/xml/XmlSource.java | 22 +- .../org/apache/beam/sdk/io/xml/JAXBCoderTest.java | 2 +- .../java/org/apache/beam/sdk/io/xml/XmlIOTest.java | 11 +- .../org/apache/beam/sdk/io/xml/XmlSourceTest.java | 44 +- sdks/java/maven-archetypes/examples/build.gradle | 2 +- sdks/java/testing/jpms-tests/build.gradle | 2 +- .../java/org/apache/beam/sdk/nexmark/Monitor.java | 13 +- .../apache/beam/sdk/nexmark/NexmarkLauncher.java | 50 +- .../apache/beam/sdk/nexmark/NexmarkOptions.java | 14 + .../beam/sdk/nexmark/queries/NexmarkQuery.java | 3 - .../sdk/nexmark/queries/NexmarkQueryModel.java | 4 +- .../apache/beam/sdk/nexmark/queries/Query10.java | 8 +- .../apache/beam/sdk/nexmark/queries/Query3.java | 191 +++-- .../apache/beam/sdk/nexmark/queries/Query5.java | 133 +++- .../beam/sdk/nexmark/NexmarkConfigurationTest.java | 2 +- .../beam/sdk/nexmark/PerfsToBigQueryTest.java | 2 +- .../nexmark/queries/BoundedSideInputJoinTest.java | 2 +- .../nexmark/queries/SessionSideInputJoinTest.java | 2 +- .../queries/sql/SqlBoundedSideInputJoinTest.java | 2 +- sdks/python/apache_beam/coders/coders.py | 71 +- .../apache_beam/coders/coders_test_common.py | 1 - sdks/python/apache_beam/coders/typecoders.py | 3 - sdks/python/apache_beam/dataframe/frames.py | 26 +- sdks/python/apache_beam/dataframe/frames_test.py | 78 +- sdks/python/apache_beam/dataframe/io.py | 263 ++++-- sdks/python/apache_beam/dataframe/io_test.py | 88 +- .../examples/cookbook/bigquery_side_input.py | 7 +- .../apache_beam/examples/cookbook/filters.py | 2 +- .../apache_beam/examples/snippets/snippets.py | 16 +- .../apache_beam/examples/snippets/snippets_test.py | 10 +- sdks/python/apache_beam/examples/sql_taxi.py | 3 +- sdks/python/apache_beam/io/concat_source.py | 9 +- sdks/python/apache_beam/io/concat_source_test.py | 8 + .../flink/flink_streaming_impulse_source_test.py | 3 +- sdks/python/apache_beam/io/gcp/bigquery.py | 2 +- .../apache_beam/io/gcp/bigquery_file_loads.py | 21 +- .../apache_beam/io/gcp/bigquery_read_internal.py | 40 +- sdks/python/apache_beam/io/gcp/bigquery_test.py | 14 +- sdks/python/apache_beam/io/gcp/pubsub.py | 8 +- sdks/python/apache_beam/io/gcp/pubsub_test.py | 3 +- sdks/python/apache_beam/io/mongodbio.py | 29 +- sdks/python/apache_beam/io/mongodbio_test.py | 41 + sdks/python/apache_beam/io/restriction_trackers.py | 2 +- sdks/python/apache_beam/metrics/execution.pxd | 1 + sdks/python/apache_beam/metrics/execution.py | 9 +- .../python/apache_beam/options/pipeline_options.py | 29 +- sdks/python/apache_beam/pipeline.py | 163 ++-- sdks/python/apache_beam/portability/common_urns.py | 1 + sdks/python/apache_beam/runners/common.pxd | 1 + sdks/python/apache_beam/runners/common.py | 36 +- .../runners/dataflow/dataflow_runner.py | 184 ++--- .../runners/dataflow/dataflow_runner_test.py | 62 ++ .../apache_beam/runners/dataflow/internal/names.py | 9 +- .../runners/dataflow/ptransform_overrides.py | 45 ++ .../apache_beam/runners/direct/direct_runner.py | 4 +- .../apache_beam/runners/direct/direct_userstate.py | 35 +- .../runners/direct/transform_evaluator.py | 8 +- sdks/python/apache_beam/runners/direct/util.py | 18 +- .../runners/direct/watermark_manager.py | 10 +- .../python/apache_beam/runners/pipeline_context.py | 2 - .../runners/portability/flink_runner_test.py | 3 + .../portability/fn_api_runner/fn_runner_test.py | 24 + .../portability/fn_api_runner/translations.py | 81 +- .../portability/fn_api_runner/translations_test.py | 18 +- .../portability/fn_api_runner/worker_handlers.py | 7 +- .../apache_beam/runners/portability/job_server.py | 10 +- .../runners/portability/job_server_test.py | 84 ++ .../runners/portability/portable_runner.py | 85 +- .../runners/portability/sdk_container_builder.py | 11 +- .../runners/portability/spark_runner_test.py | 5 +- sdks/python/apache_beam/runners/runner.py | 24 +- .../apache_beam/runners/worker/bundle_processor.py | 41 +- .../apache_beam/runners/worker/operations.py | 6 +- sdks/python/apache_beam/transforms/core.py | 145 +--- sdks/python/apache_beam/transforms/environments.py | 26 +- sdks/python/apache_beam/transforms/external.py | 24 +- sdks/python/apache_beam/transforms/ptransform.py | 46 +- sdks/python/apache_beam/transforms/trigger.py | 39 +- sdks/python/apache_beam/transforms/trigger_test.py | 10 +- sdks/python/apache_beam/transforms/userstate.py | 55 +- .../apache_beam/transforms/userstate_test.py | 216 ++++- sdks/python/apache_beam/transforms/util.py | 123 ++- sdks/python/apache_beam/transforms/util_test.py | 65 ++ .../typehints/native_type_compatibility.py | 7 - .../apache_beam/typehints/sharded_key_type.py | 31 +- .../apache_beam/typehints/sharded_key_type_test.py | 35 +- sdks/python/apache_beam/version.py | 2 +- sdks/python/container/common.gradle | 2 + sdks/python/setup.py | 6 +- sdks/python/test-suites/dataflow/build.gradle | 24 + sdks/python/test-suites/gradle.properties | 5 + sdks/python/test-suites/portable/common.gradle | 34 +- settings.gradle | 5 + start-build-env.sh | 16 +- website/.gitignore | 23 - website/www/package.json | 2 +- website/www/site/config.toml | 2 +- website/www/site/content/en/blog/beam-2.13.0.md | 2 +- website/www/site/content/en/blog/beam-2.21.0.md | 2 +- website/www/site/content/en/blog/beam-2.22.0.md | 2 +- website/www/site/content/en/blog/beam-2.23.0.md | 4 +- website/www/site/content/en/blog/beam-2.24.0.md | 4 +- website/www/site/content/en/blog/beam-2.25.0.md | 4 +- website/www/site/content/en/blog/beam-2.26.0.md | 65 ++ website/www/site/content/en/blog/beam-2.27.0.md | 69 ++ .../en/blog/dataframe-api-preview-available.md | 178 +++++ .../site/content/en/blog/pattern-match-beam-sql.md | 2 +- .../en/blog/splittable-do-fn-is-available.md | 91 +++ website/www/site/content/en/contribute/_index.md | 2 +- .../site/content/en/contribute/release-guide.md | 7 +- .../en/documentation/dsls/dataframes/overview.md | 131 +++ .../dsls/sql/extensions/create-external-table.md | 15 +- .../content/en/documentation/dsls/sql/overview.md | 2 +- .../documentation/io/built-in/google-bigquery.md | 15 +- .../en/documentation/io/built-in/snowflake.md | 6 +- .../content/en/documentation/programming-guide.md | 4 +- .../content/en/documentation/runners/direct.md | 112 +-- .../site/content/en/documentation/runners/flink.md | 42 +- .../site/content/en/documentation/runners/samza.md | 5 + .../en/documentation/runtime/environments.md | 305 ++++--- .../site/content/en/get-started/beam-overview.md | 5 +- .../www/site/content/en/get-started/downloads.md | 28 +- .../www/site/content/en/get-started/from-spark.md | 268 +++++++ website/www/site/data/authors.yml | 38 +- website/www/site/data/capability_matrix.yaml | 714 +++++++++++++++-- .../www/site/data/capability_matrix_snapshot.yaml | 195 +++++ website/www/site/layouts/blog/single.html | 1 + .../partials/section-menu/en/get-started.html | 7 +- .../layouts/partials/section-menu/en/sdks.html | 17 +- website/www/site/static/.htaccess | 2 +- 941 files changed, 26264 insertions(+), 6811 deletions(-)
