This is an automated email from the ASF dual-hosted git repository.
damccorm pushed a change to branch users/damccorm/prismByDefault
in repository https://gitbox.apache.org/repos/asf/beam.git
from 8f76ca4022d Wait until finish for test pipelines
add aee8a84115c support MicrosInstant for long type (#35065)
add bb7f4454bd7 Clean up GCP Resources (Pubsub) (#34141)
add bbf7124fd5b added credit karma financial insights for use cases
(#35039)
add 229fec66347 fixed the image link for credit karma (#35094)
add be6b3142a07 Bump cloud.google.com/go/spanner from 1.80.0 to 1.82.0 in
/sdks (#35077)
add 697e054b8e8 fix prism runner race condition (#35024)
add ac501c64127 add input data file for example tests (#35054)
add 16b43353761 Move recently added yaml tests to tmp folder to lower
precommit runtime (#35096)
add 2e956263496 Prism: Fix event-time timer firing earlier than the time
set. (#35089)
add 8439d33b816 Delete unused page (#35104)
add f9dca88eb62 Fix the issue of timer clearing not working across bundle
(#35106)
add 1f6ec353e06 update container tag (#35097)
add b69f23f6818 Fix for issue 34772 to include user provided providers
(#35033)
add 6ae6f38942e update test your pipeline with TestPipeline with Rule
(#35105)
add 8995f27b02c Add license for kafka-python (#35116)
add 7eb7b9b2a3a Update dataflow containers (#35118)
add 7afeb306feb [Website] add akvelon case study (#34943)
add c6c815d3de3 added the rate test for GenerateSequence (#35108)
add 792c865a939 Re-enable logging after importing vllm. (#35103)
add 9a7914b81d2 Deprecate Java 8 (#35064)
add ab191181d9d Remove beam college banners (#35123)
add 49928ea7ab8 feat: change text (#35130)
add 50ce6c11961 Update Custom Remote Inference example to use
RemoteModelHandler (#35066)
add 469168ed55e Remove Java 8 container (#35125)
add 64ea938877c add extra_transforms block documentation to chain
transform documentation (#35101)
add a33003ee425 add note about testing (#35075)
add 8c43e8c4cf2 [Website] update akvelon case study: update text and fix
landing page (#35133)
add 28d209cc632 Fix PostCommit Python Xlang IO Dataflow job (#35131)
add 439b288a2f7 Bump google.golang.org/grpc from 1.72.0 to 1.72.2 in /sdks
(#35113)
add 138fad37427 Bump cloud.google.com/go/bigquery from 1.67.0 to 1.69.0 in
/sdks (#35061)
add ebf45a83822 Add known issues. (#35138)
add 1e1e3cc7893 Bump @octokit/plugin-paginate-rest and @octokit/rest
(#34167)
add bb2a3ec03a7 Explicitly handle singleton iterators instead of using
helper and catching exceptions which may be from generating iterable (#35124)
add 48ebe6170f3 Build last snapshot against RC00 tag instead of release
branch (#35142)
add 9469674825a Bump nodemailer from 6.7.5 to 6.9.9 in
/scripts/ci/issue-report (#35143)
add 8f32ee179fd Fix tests affected by Java 8 container turn down (#35145)
add a4707ab35a3 Bump github.com/aws/aws-sdk-go-v2/service/s3 in /sdks
(#35146)
add dd51c4cba10 fix jdbc transform validation (#35141)
add 4c607a70c26 Fix Java Example ARM PostCommit
add 14ac3711e65 Fix Java Example ARM PostCommit (#35154)
add b7f2e161155 fix: add missed word (#35163)
add e6f50a84501 Add postcommit yaml xlang workflow and split tests
accordingly (#35119)
add a856eb9916b Replace usages of deprecated pkg_resources package (#35153)
add c87b3d6ddac Improve error message when accidentally using
PBegin/Pipeline (#35156)
add 9e7c4393287 add friendly error message for when transform is applied
to no output (#35160)
add a96938664a7 Add warning if temp location bucket has soft delete
enabled for Go SD… (#34996)
add 3f0671d6a94 Constrain DequeCoder type correctly, as it does not
support nulls
add 5ada3c1dbe7 Merge pull request #35084: Constrain DequeCoder type
correctly, as it does not support nulls
add e1d047ed098 Do not overwrite class states if a cached dynamic class is
returned in cloudpickle.load (#35063)
add 8b957b1f207 Make SDK harness change effective on Iceberg Dataflow test
(#35173)
add 2330c800eb9 Fix beam_PostCommit_Java_Examples_Dataflow_V2 (#35172)
add a48cca263a3 [YAML]: Update postgres IT test and readme (#35169)
add 0004b45a503 Bump Java beam-master container (#35170)
add 94d32611cac Make WindowedValue a public interface
add 479f0d68042 Run integration tests for moving WindowedValue and making
public
add deff583cf6e Make SDK harness change effective on Iceberg Dataflow test
add acafd37ba45 Merge pull request #35159: Make WindowedValue a public
interface
add 18a3fcfc06c Add timer tests to make sure event-time timer firing at
the right time. (#35109)
add e8097fbf19f change to ubuntu-20.04 (#35182)
add 82bda513eca Fix IntelliJ sync project failure due to circular Beam
dependency (#35167)
add a77582424c8 Update workflows categories (#35162)
add 444e37ec2a2 Add cloudpickle coder. (#35166)
add feb9cda6863 Move examples from sql package (#35183)
add 5dba0dba804 Fix the beam interactive install problem when on Google
Colab (#35148)
add 46f734dd72b Bump github.com/docker/docker in /sdks (#35112)
add 4ca0df664bc Bump github.com/nats-io/nats-server/v2 from 2.11.3 to
2.11.4 in /sdks (#35161)
add 84706d97eac Touch JPMS test trigger file
add bc652a1d947 Use staged SDK harness & Dataflow worker jar in JPMS tests
add c37785b18d5 Merge pull request #35181: Use staged SDK harness &
Dataflow worker jar in JPMS tests
add b85c28b3bf9 Bump cloud.google.com/go/storage from 1.52.0 to 1.55.0 in
/sdks (#35114)
add 17d98237950 Bump github.com/nats-io/nats.go from 1.42.0 to 1.43.0 in
/sdks (#35147)
add ad7c151b9a5 Move non-dill specific code out of dill_pickler.py (#35139)
add b4e5e396291 Fix some lints introduced in a recent PR. (#35193)
add f7fc608af9a small filesystem fixes (#34956)
add 74512d9d04d [YAML] Add a spec provider for transforms taking
specifiable arguments (#35187)
add 476f332d7e3 Touch trigger files to test WindowedValueReceiver in
runners
add b84cf002397 Introduce WindowedValue receivers and consolidate runner
code to them
add b6d4fdcd592 Merge pull request #35177: Introduce WindowedValue
receivers and consolidate runner code to them
add da446f4e7b1 Eliminate nullness errors from ByteBuddyDoFnInvokerFactory
and DoFnOutputReceivers
add 7995a5e0dd3 Merge pull request #35020: Fix nullability in
ByteBuddyDoFnInvokerFactory and DoFnOutputReceivers
add 3c07e82872b Fix null check when fetching driverJars from value provider
add 2f79185801e Merge pull request #35184: Fix null check when fetching
driverJars from value provider
add 7b2134c68c5 Fix PostCommit Python ValidatesRunner Samza / Spark jobs
(#35210)
add df54bf439b2 Update pypi documentation 30145 (#34329)
add 7b43e33e560 Merge branch 'master' of https://github.com/apache/beam
into users/damccorm/prismByDefault
add a10a8822345 fix test_always
No new revisions were added by this update.
Summary of changes:
.../IO_Iceberg_Integration_Tests.json | 3 +-
.../IO_Iceberg_Integration_Tests_Dataflow.json | 3 +-
.../beam_PostCommit_Java_DataflowV1.json | 4 +-
.../beam_PostCommit_Java_DataflowV2.json | 4 +-
...eam_PostCommit_Java_Examples_Dataflow_ARM.json} | 0
...eam_PostCommit_Java_Examples_Dataflow_Java.json | 1 +
.../beam_PostCommit_Java_Examples_Dataflow_V2.json | 2 +-
.../beam_PostCommit_Java_Jpms_Dataflow_Java11.json | 3 +
.../beam_PostCommit_Java_Jpms_Flink_Java11.json | 2 +-
...m_PostCommit_Java_ValidatesRunner_Dataflow.json | 3 +-
...it_Java_ValidatesRunner_Dataflow_Streaming.json | 3 +-
...ostCommit_Java_ValidatesRunner_Dataflow_V2.json | 3 +-
...eam_PostCommit_Java_ValidatesRunner_Direct.json | 4 +-
...beam_PostCommit_Java_ValidatesRunner_Flink.json | 4 +-
...beam_PostCommit_Java_ValidatesRunner_Samza.json | 4 +-
...beam_PostCommit_Java_ValidatesRunner_Spark.json | 4 +-
...a_ValidatesRunner_SparkStructuredStreaming.json | 4 +-
.../beam_PostCommit_Java_ValidatesRunner_ULR.json | 3 +-
.../trigger_files/beam_PostCommit_XVR_Direct.json | 2 +-
.../trigger_files/beam_PostCommit_XVR_Samza.json | 2 +-
...json => beam_PostCommit_Yaml_Xlang_Direct.json} | 2 +-
.github/workflows/README.md | 1 +
.github/workflows/beam_PostCommit_Go_VR_Samza.yml | 6 +-
.../beam_PostCommit_Java_Examples_Dataflow_V2.yml | 8 +-
.github/workflows/beam_PostCommit_XVR_Direct.yml | 17 +-
.github/workflows/beam_PostCommit_XVR_Flink.yml | 17 +-
.github/workflows/beam_PostCommit_XVR_Samza.yml | 19 +-
.github/workflows/beam_PostCommit_XVR_Spark3.yml | 17 +-
...T.yml => beam_PostCommit_Yaml_Xlang_Direct.yml} | 29 +-
.../workflows/beam_PreCommit_Yaml_Xlang_Direct.yml | 2 +-
.../workflows/beam_Publish_Beam_SDK_Snapshots.yml | 1 -
.../xlang_KafkaIO_Python.txt | 2 +-
.../github/github_runs_prefetcher/code/config.yaml | 59 +--
.test-infra/tools/requirements.txt | 4 +-
.test-infra/tools/stale_cleaner.py | 306 +++++++++++++
.test-infra/tools/test_stale_cleaner.py | 491 +++++++++++++++++++++
CHANGES.md | 11 +-
README.md | 4 +
build.gradle.kts | 2 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 21 +-
contributor-docs/release-guide.md | 2 +-
examples/java/cdap/hubspot/build.gradle | 9 +-
examples/java/cdap/salesforce/build.gradle | 9 +-
.../examples/{sql => }/SchemaTransformExample.java | 2 +-
.../examples/{sql => }/SqlTransformExample.java | 2 +-
.../beam-ml/custom_remote_inference.ipynb | 360 ++++++++-------
.../org/apache/beam/runners/core/DoFnRunner.java | 2 +-
.../org/apache/beam/runners/core/DoFnRunners.java | 13 +-
.../core/GroupAlsoByWindowViaWindowSetNewDoFn.java | 36 +-
.../runners/core/GroupByKeyViaGroupByKeyOnly.java | 4 +-
.../apache/beam/runners/core/KeyedWorkItem.java | 2 +-
.../beam/runners/core/KeyedWorkItemCoder.java | 4 +-
.../apache/beam/runners/core/KeyedWorkItems.java | 2 +-
.../runners/core/LateDataDroppingDoFnRunner.java | 5 +-
.../apache/beam/runners/core/LateDataUtils.java | 2 +-
...TimeBoundedSplittableProcessElementInvoker.java | 22 +-
.../beam/runners/core/OutputWindowedValue.java | 45 --
.../apache/beam/runners/core/ProcessFnRunner.java | 2 +-
.../runners/core/PushbackSideInputDoFnRunner.java | 2 +-
.../apache/beam/runners/core/ReduceFnRunner.java | 11 +-
.../apache/beam/runners/core/SideInputHandler.java | 2 +-
.../apache/beam/runners/core/SimpleDoFnRunner.java | 17 +-
.../core/SimplePushbackSideInputDoFnRunner.java | 2 +-
.../core/SplittableParDoViaKeyedWorkItems.java | 5 +-
.../core/SplittableProcessElementInvoker.java | 2 +-
.../beam/runners/core/StatefulDoFnRunner.java | 5 +-
.../beam/runners/core/KeyedWorkItemCoderTest.java | 9 +-
.../core/LateDataDroppingDoFnRunnerTest.java | 5 +-
...BoundedSplittableProcessElementInvokerTest.java | 27 +-
.../beam/runners/core/ReduceFnRunnerTest.java | 2 +-
.../apache/beam/runners/core/ReduceFnTester.java | 38 +-
.../beam/runners/core/SideInputHandlerTest.java | 5 +-
.../beam/runners/core/SimpleDoFnRunnerTest.java | 44 +-
.../SimplePushbackSideInputDoFnRunnerTest.java | 28 +-
.../runners/core/SplittableParDoProcessFnTest.java | 43 +-
.../beam/runners/core/StatefulDoFnRunnerTest.java | 74 ++--
.../apache/beam/runners/core/WindowMatchers.java | 2 +-
.../beam/runners/core/WindowMatchersTest.java | 6 +-
.../core/triggers/TriggerStateMachineTester.java | 6 +-
.../runners/direct/AbstractModelEnforcement.java | 2 +-
.../direct/BoundedReadEvaluatorFactory.java | 7 +-
.../beam/runners/direct/CloningBundleFactory.java | 2 +-
.../beam/runners/direct/CommittedBundle.java | 2 +-
.../direct/CreateViewNoopEvaluatorFactory.java | 2 +-
.../runners/direct/DirectTransformExecutor.java | 2 +-
...LifecycleManagerRemovingTransformEvaluator.java | 2 +-
.../runners/direct/FlattenEvaluatorFactory.java | 2 +-
.../direct/GroupAlsoByWindowEvaluatorFactory.java | 35 +-
.../direct/GroupByKeyOnlyEvaluatorFactory.java | 5 +-
.../direct/ImmutabilityCheckingBundleFactory.java | 2 +-
.../direct/ImmutabilityEnforcementFactory.java | 2 +-
.../runners/direct/ImmutableListBundleFactory.java | 2 +-
.../runners/direct/ImpulseEvaluatorFactory.java | 8 +-
.../beam/runners/direct/ModelEnforcement.java | 2 +-
.../beam/runners/direct/MultiStepCombine.java | 2 +-
.../beam/runners/direct/PCollectionViewWriter.java | 2 +-
.../apache/beam/runners/direct/ParDoEvaluator.java | 13 +-
.../runners/direct/ParDoMultiOverrideFactory.java | 7 +-
.../direct/PassthroughTransformEvaluator.java | 2 +-
.../beam/runners/direct/QuiescenceDriver.java | 4 +-
.../beam/runners/direct/SideInputContainer.java | 2 +-
.../SplittableProcessElementsEvaluatorFactory.java | 35 +-
.../direct/StatefulParDoEvaluatorFactory.java | 2 +-
.../beam/runners/direct/StepTransformResult.java | 2 +-
.../runners/direct/TestStreamEvaluatorFactory.java | 9 +-
.../beam/runners/direct/TransformEvaluator.java | 2 +-
.../beam/runners/direct/TransformResult.java | 2 +-
.../direct/UnboundedReadEvaluatorFactory.java | 11 +-
.../beam/runners/direct/UncommittedBundle.java | 2 +-
.../beam/runners/direct/ViewEvaluatorFactory.java | 2 +-
.../runners/direct/WindowEvaluatorFactory.java | 5 +-
.../direct/BoundedReadEvaluatorFactoryTest.java | 11 +-
.../runners/direct/CloningBundleFactoryTest.java | 21 +-
.../beam/runners/direct/CommittedResultTest.java | 4 +-
.../direct/DirectTransformExecutorTest.java | 19 +-
...cycleManagerRemovingTransformEvaluatorTest.java | 9 +-
.../beam/runners/direct/EvaluationContextTest.java | 11 +-
.../direct/FlattenEvaluatorFactoryTest.java | 26 +-
.../direct/GroupByKeyEvaluatorFactoryTest.java | 27 +-
.../direct/GroupByKeyOnlyEvaluatorFactoryTest.java | 27 +-
.../ImmutabilityCheckingBundleFactoryTest.java | 19 +-
.../direct/ImmutabilityEnforcementFactoryTest.java | 9 +-
.../direct/ImmutableListBundleFactoryTest.java | 31 +-
.../direct/ImpulseEvaluatorFactoryTest.java | 5 +-
.../direct/KeyedPValueTrackingVisitorTest.java | 9 +-
.../beam/runners/direct/ParDoEvaluatorTest.java | 13 +-
.../runners/direct/SideInputContainerTest.java | 31 +-
.../direct/StatefulParDoEvaluatorFactoryTest.java | 5 +-
.../direct/TestStreamEvaluatorFactoryTest.java | 15 +-
.../direct/UnboundedReadEvaluatorFactoryTest.java | 13 +-
.../runners/direct/ViewEvaluatorFactoryTest.java | 7 +-
.../beam/runners/direct/WatermarkManagerTest.java | 32 +-
.../runners/direct/WindowEvaluatorFactoryTest.java | 9 +-
.../FlinkBatchPortablePipelineTranslator.java | 13 +-
.../flink/FlinkBatchTransformTranslators.java | 16 +-
.../flink/FlinkBatchTranslationContext.java | 7 +-
.../FlinkStreamingAggregationsTranslators.java | 15 +-
.../FlinkStreamingPortablePipelineTranslator.java | 39 +-
.../flink/FlinkStreamingTransformTranslators.java | 25 +-
.../flink/FlinkStreamingTranslationContext.java | 5 +-
.../flink/adapter/BeamFlinkDataSetAdapter.java | 7 +-
.../flink/adapter/BeamFlinkDataStreamAdapter.java | 7 +-
.../flink/metrics/DoFnRunnerWithMetricsUpdate.java | 2 +-
.../functions/AbstractFlinkCombineRunner.java | 2 +-
.../translation/functions/FlinkAssignContext.java | 5 +-
.../translation/functions/FlinkAssignWindows.java | 5 +-
.../translation/functions/FlinkDoFnFunction.java | 53 ++-
.../functions/FlinkExecutableStageFunction.java | 2 +-
.../FlinkExecutableStagePruningFunction.java | 2 +-
.../functions/FlinkExplodeWindowsFunction.java | 2 +-
.../FlinkMergingNonShuffleReduceFunction.java | 2 +-
.../functions/FlinkMultiOutputPruningFunction.java | 2 +-
.../functions/FlinkNonMergingReduceFunction.java | 5 +-
.../functions/FlinkPartialReduceFunction.java | 2 +-
.../translation/functions/FlinkReduceFunction.java | 2 +-
.../functions/FlinkStatefulDoFnFunction.java | 7 +-
.../functions/HashingFlinkCombineRunner.java | 5 +-
.../functions/ImpulseSourceFunction.java | 5 +-
.../functions/SideInputInitializer.java | 2 +-
.../functions/SingleWindowFlinkCombineRunner.java | 5 +-
.../functions/SortingFlinkCombineRunner.java | 11 +-
.../flink/translation/types/KvKeySelector.java | 2 +-
.../translation/types/WindowedKvKeySelector.java | 2 +-
.../translation/wrappers/ImpulseInputFormat.java | 5 +-
.../translation/wrappers/SourceInputFormat.java | 5 +-
.../wrappers/streaming/DoFnOperator.java | 8 +-
.../streaming/ExecutableStageDoFnOperator.java | 11 +-
.../streaming/KvToFlinkKeyKeySelector.java | 2 +-
.../streaming/PartialReduceBundleOperator.java | 5 +-
.../wrappers/streaming/SdfFlinkKeyKeySelector.java | 2 +-
.../wrappers/streaming/SingletonKeyedWorkItem.java | 2 +-
.../streaming/SingletonKeyedWorkItemCoder.java | 7 +-
.../wrappers/streaming/SplittableDoFnOperator.java | 32 +-
.../wrappers/streaming/WindowDoFnOperator.java | 5 +-
.../wrappers/streaming/WorkItemKeySelector.java | 2 +-
.../wrappers/streaming/io/DedupingOperator.java | 2 +-
.../streaming/io/StreamingImpulseSource.java | 5 +-
.../wrappers/streaming/io/TestStreamSource.java | 5 +-
.../streaming/io/UnboundedSourceWrapper.java | 5 +-
.../io/source/bounded/FlinkBoundedSource.java | 2 +-
.../source/bounded/FlinkBoundedSourceReader.java | 5 +-
.../io/source/unbounded/FlinkUnboundedSource.java | 2 +-
.../unbounded/FlinkUnboundedSourceReader.java | 5 +-
.../streaming/stableinput/BufferedElements.java | 2 +-
.../streaming/stableinput/BufferingDoFnRunner.java | 2 +-
.../runners/flink/FlinkPipelineOptionsTest.java | 9 +-
.../flink/streaming/BoundedSourceRestoreTest.java | 2 +-
.../FlinkExecutableStageFunctionTest.java | 15 +-
.../functions/ImpulseSourceFunctionTest.java | 2 +-
.../wrappers/streaming/DedupingOperatorTest.java | 18 +-
.../wrappers/streaming/DoFnOperatorTest.java | 342 +++++++-------
.../streaming/ExecutableStageDoFnOperatorTest.java | 45 +-
.../wrappers/streaming/StreamRecordStripper.java | 2 +-
.../wrappers/streaming/WindowDoFnOperatorTest.java | 19 +-
.../streaming/io/UnboundedSourceWrapperTest.java | 2 +-
.../bounded/FlinkBoundedSourceReaderTest.java | 2 +-
.../unbounded/FlinkUnboundedSourceReaderTest.java | 2 +-
.../stableinput/BufferedElementsTest.java | 8 +-
.../stableinput/BufferingDoFnRunnerTest.java | 4 +-
.../google-cloud-dataflow-java/arm/build.gradle | 8 +-
runners/google-cloud-dataflow-java/build.gradle | 20 +-
.../dataflow/BatchStatefulParDoOverrides.java | 7 +-
.../beam/runners/dataflow/BatchViewOverrides.java | 18 +-
.../dataflow/DataflowPipelineTranslator.java | 6 +-
.../beam/runners/dataflow/DataflowRunner.java | 26 +-
.../dataflow/util/CloudObjectTranslators.java | 2 +-
.../beam/runners/dataflow/util/CloudObjects.java | 2 +-
.../beam/runners/dataflow/dataflow.properties | 3 +
.../runners/dataflow/BatchViewOverridesTest.java | 84 ++--
.../runners/dataflow/util/CloudObjectsTest.java | 4 +-
.../worker/AssignWindowsParDoFnFactory.java | 5 +-
.../worker/BatchModeUngroupingParDoFn.java | 2 +-
.../CreateIsmShardKeyAndSortKeyDoFnFactory.java | 2 +-
.../dataflow/worker/DataflowOutputCounter.java | 2 +-
.../worker/DataflowPortabilityPCollectionView.java | 2 +-
.../dataflow/worker/DataflowProcessFnRunner.java | 5 +-
.../runners/dataflow/worker/DoFnRunnerFactory.java | 4 +-
.../dataflow/worker/GroupAlsoByWindowFn.java | 6 +-
.../dataflow/worker/GroupAlsoByWindowFnRunner.java | 39 +-
.../worker/GroupAlsoByWindowParDoFnFactory.java | 4 +-
.../dataflow/worker/GroupAlsoByWindowsParDoFn.java | 30 +-
.../dataflow/worker/GroupingShuffleReader.java | 4 +-
.../worker/GroupingShuffleReaderFactory.java | 2 +-
...ingShuffleReaderWithFaultyBytesReadCounter.java | 2 +-
.../worker/IntrinsicMapTaskExecutorFactory.java | 2 +-
.../beam/runners/dataflow/worker/IsmReader.java | 2 +-
.../runners/dataflow/worker/IsmReaderFactory.java | 4 +-
.../runners/dataflow/worker/IsmReaderImpl.java | 2 +-
.../dataflow/worker/IsmSideInputReader.java | 2 +-
.../beam/runners/dataflow/worker/IsmSink.java | 2 +-
.../runners/dataflow/worker/IsmSinkFactory.java | 4 +-
.../worker/PairWithConstantKeyDoFnFactory.java | 2 +-
.../dataflow/worker/PartialGroupByKeyParDoFns.java | 13 +-
.../dataflow/worker/PartitioningShuffleReader.java | 4 +-
.../worker/PartitioningShuffleReaderFactory.java | 2 +-
.../runners/dataflow/worker/PubsubDynamicSink.java | 2 +-
.../beam/runners/dataflow/worker/PubsubReader.java | 7 +-
.../beam/runners/dataflow/worker/PubsubSink.java | 4 +-
.../ReifyTimestampAndWindowsParDoFnFactory.java | 7 +-
.../beam/runners/dataflow/worker/ShuffleSink.java | 4 +-
.../dataflow/worker/ShuffleSinkFactory.java | 2 +-
.../dataflow/worker/SimpleDoFnRunnerFactory.java | 4 +-
.../runners/dataflow/worker/SimpleParDoFn.java | 10 +-
.../worker/SplittableProcessFnFactory.java | 33 +-
.../StreamingGroupAlsoByWindowReshuffleFn.java | 13 +-
.../StreamingGroupAlsoByWindowViaWindowSetFn.java | 4 +-
.../StreamingKeyedWorkItemSideInputDoFnRunner.java | 2 +-
.../StreamingPCollectionViewWriterDoFnFactory.java | 2 +-
.../StreamingPCollectionViewWriterParDoFn.java | 2 +-
.../worker/StreamingSideInputDoFnRunner.java | 2 +-
.../dataflow/worker/StreamingSideInputFetcher.java | 5 +-
.../worker/ToIsmRecordForMultimapDoFnFactory.java | 2 +-
.../dataflow/worker/UngroupedWindmillReader.java | 9 +-
.../runners/dataflow/worker/ValuesDoFnFactory.java | 2 +-
.../dataflow/worker/WindmillKeyedWorkItem.java | 5 +-
.../worker/WindmillReaderIteratorBase.java | 2 +-
.../beam/runners/dataflow/worker/WindmillSink.java | 4 +-
.../dataflow/worker/WindowingWindmillReader.java | 4 +-
.../dataflow/worker/WorkerCustomSources.java | 7 +-
.../dataflow/worker/status/WorkerStatusPages.java | 5 +-
.../util/BatchGroupAlsoByWindowAndCombineFn.java | 21 +-
.../worker/util/BatchGroupAlsoByWindowFn.java | 2 +-
.../util/BatchGroupAlsoByWindowReshuffleFn.java | 12 +-
.../util/BatchGroupAlsoByWindowViaIteratorsFn.java | 19 +-
.../BatchGroupAlsoByWindowViaOutputBufferFn.java | 6 +-
.../dataflow/worker/util/MemoryMonitor.java | 13 +-
.../dataflow/worker/util/ValueInEmptyWindows.java | 9 +-
.../processing/ComputationWorkExecutorFactory.java | 8 +-
.../dataflow/worker/AvroByteReaderFactoryTest.java | 6 +-
.../dataflow/worker/AvroByteSinkFactoryTest.java | 4 +-
.../worker/CombineValuesFnFactoryTest.java | 42 +-
...CreateIsmShardKeyAndSortKeyDoFnFactoryTest.java | 2 +-
.../DataflowPortabilityPCollectionViewTest.java | 2 +-
.../dataflow/worker/DefaultParDoFnFactoryTest.java | 4 +-
.../dataflow/worker/GroupingShuffleReaderTest.java | 21 +-
.../IntrinsicMapTaskExecutorFactoryTest.java | 6 +-
.../dataflow/worker/IsmReaderFactoryTest.java | 8 +-
.../runners/dataflow/worker/IsmReaderTest.java | 2 +-
.../dataflow/worker/IsmSideInputReaderTest.java | 53 +--
.../beam/runners/dataflow/worker/IsmSinkTest.java | 2 +-
.../worker/PairWithConstantKeyDoFnFactoryTest.java | 2 +-
.../worker/PartialGroupByKeyParDoFnsTest.java | 75 ++--
.../worker/PartitioningShuffleReaderTest.java | 25 +-
.../dataflow/worker/PubsubDynamicSinkTest.java | 11 +-
.../runners/dataflow/worker/PubsubReaderTest.java | 11 +-
.../runners/dataflow/worker/PubsubSinkTest.java | 20 +-
.../runners/dataflow/worker/ReaderFactoryTest.java | 5 +-
.../runners/dataflow/worker/ReaderTestUtils.java | 2 +-
...ReifyTimestampAndWindowsParDoFnFactoryTest.java | 7 +-
.../dataflow/worker/ShuffleReaderFactoryTest.java | 4 +-
.../dataflow/worker/ShuffleSinkFactoryTest.java | 14 +-
.../runners/dataflow/worker/ShuffleSinkTest.java | 15 +-
.../runners/dataflow/worker/SimpleParDoFnTest.java | 48 +-
.../worker/StreamingDataflowWorkerTest.java | 19 +-
.../worker/StreamingGroupAlsoByWindowFnsTest.java | 41 +-
...reamingGroupAlsoByWindowsReshuffleDoFnTest.java | 11 +-
...eamingKeyedWorkItemSideInputDoFnRunnerTest.java | 13 +-
...eamingPCollectionViewWriterDoFnFactoryTest.java | 4 +-
.../worker/StreamingSideInputDoFnRunnerTest.java | 14 +-
.../worker/StreamingSideInputFetcherTest.java | 5 +-
.../ToIsmRecordForMultimapDoFnFactoryTest.java | 2 +-
.../worker/UngroupedShuffleReaderTest.java | 7 +-
.../dataflow/worker/UserParDoFnFactoryTest.java | 18 +-
.../dataflow/worker/ValuesDoFnFactoryTest.java | 2 +-
.../dataflow/worker/WindmillKeyedWorkItemTest.java | 8 +-
.../worker/WindmillReaderIteratorBaseTest.java | 5 +-
.../dataflow/worker/WorkerCustomSourcesTest.java | 4 +-
.../graph/LengthPrefixUnknownCodersTest.java | 15 +-
.../BatchGroupAlsoByWindowReshuffleDoFnTest.java | 18 +-
...gGroupAlsoByWindowsViaOutputBufferDoFnTest.java | 2 +-
.../worker/util/GroupAlsoByWindowProperties.java | 93 ++--
.../dataflow/worker/util/ListOutputManager.java | 43 +-
.../dataflow/worker/util/MemoryMonitorTest.java | 6 -
.../control/BundleCheckpointHandlers.java | 5 +-
.../control/ProcessBundleDescriptors.java | 4 +-
.../translation/BatchSideInputHandlerFactory.java | 2 +-
.../translation/PipelineTranslatorUtils.java | 9 +-
.../beam/runners/fnexecution/wire/WireCoders.java | 7 +-
.../fnexecution/control/RemoteExecutionTest.java | 11 +-
.../fnexecution/control/SdkHarnessClientTest.java | 19 +-
.../fnexecution/data/GrpcDataServiceTest.java | 23 +-
.../BatchSideInputHandlerFactoryTest.java | 30 +-
.../runners/fnexecution/wire/CommonCoderTest.java | 12 +-
.../wire/LengthPrefixUnknownCodersTest.java | 14 +-
.../org/apache/beam/runners/jet/DAGBuilder.java | 7 +-
.../beam/runners/jet/JetTransformTranslators.java | 5 +-
.../beam/runners/jet/JetTranslationContext.java | 8 +-
.../java/org/apache/beam/runners/jet/Utils.java | 18 +-
.../runners/jet/processors/AbstractParDoP.java | 9 +-
.../beam/runners/jet/processors/AssignWindowP.java | 7 +-
.../runners/jet/processors/BoundedSourceP.java | 5 +-
.../beam/runners/jet/processors/FlattenP.java | 2 +-
.../beam/runners/jet/processors/ImpulseP.java | 4 +-
.../runners/jet/processors/StatefulParDoP.java | 2 +-
.../runners/jet/processors/UnboundedSourceP.java | 4 +-
.../apache/beam/runners/jet/processors/ViewP.java | 7 +-
.../beam/runners/jet/processors/WindowGroupP.java | 33 +-
.../org/apache/beam/runners/jet/TestStreamP.java | 4 +-
.../java/org/apache/beam/runners/local/Bundle.java | 2 +-
.../runners/samza/adapter/BoundedSourceSystem.java | 5 +-
.../samza/adapter/UnboundedSourceSystem.java | 5 +-
.../samza/metrics/DoFnRunnerWithMetrics.java | 2 +-
.../runners/samza/metrics/SamzaGBKMetricOp.java | 2 +-
.../beam/runners/samza/metrics/SamzaMetricOp.java | 2 +-
.../runners/samza/runtime/AsyncDoFnRunner.java | 2 +-
.../samza/runtime/ClassicBundleManager.java | 2 +-
.../apache/beam/runners/samza/runtime/DoFnOp.java | 27 +-
.../runtime/DoFnRunnerWithKeyedInternals.java | 2 +-
.../runners/samza/runtime/FutureCollector.java | 2 +-
.../runners/samza/runtime/FutureCollectorImpl.java | 2 +-
.../beam/runners/samza/runtime/GroupByKeyOp.java | 8 +-
.../runners/samza/runtime/KvToKeyedWorkItemOp.java | 2 +-
.../org/apache/beam/runners/samza/runtime/Op.java | 2 +-
.../beam/runners/samza/runtime/OpAdapter.java | 2 +-
.../beam/runners/samza/runtime/OpEmitter.java | 2 +-
.../beam/runners/samza/runtime/OpMessage.java | 2 +-
.../samza/runtime/OutputManagerFactory.java | 8 +-
.../beam/runners/samza/runtime/PortableDoFnOp.java | 2 +-
.../runners/samza/runtime/SamzaAssignContext.java | 2 +-
.../runners/samza/runtime/SamzaDoFnRunners.java | 11 +-
.../samza/runtime/SingletonKeyedWorkItem.java | 2 +-
.../SplittableParDoProcessKeyedElementsOp.java | 32 +-
.../beam/runners/samza/runtime/WindowAssignOp.java | 5 +-
.../samza/translation/GroupByKeyTranslator.java | 9 +-
.../translation/ParDoBoundMultiTranslator.java | 17 +-
.../runners/samza/translation/ReadTranslator.java | 2 +-
.../samza/translation/ReshuffleTranslator.java | 5 +-
.../translation/SamzaImpulseSystemFactory.java | 4 +-
.../translation/SamzaPublishViewTranslator.java | 2 +-
.../translation/SamzaTestStreamSystemFactory.java | 5 +-
.../translation/SplittableParDoTranslators.java | 2 +-
.../samza/translation/TranslationContext.java | 5 +-
.../beam/runners/samza/util/SamzaCoders.java | 5 +-
.../samza/util/SamzaPipelineTranslatorUtils.java | 6 +-
.../beam/runners/samza/util/WindowUtils.java | 8 +-
.../runners/samza/adapter/TestSourceHelpers.java | 4 +-
.../TestSamzaRunnerWithTransformMetrics.java | 11 +-
.../runners/samza/runtime/AsyncDoFnRunnerTest.java | 7 +-
.../samza/runtime/ClassicBundleManagerTest.java | 2 +-
.../samza/runtime/FutureCollectorImplTest.java | 2 +-
.../io/BoundedDatasetFactory.java | 4 +-
.../translation/EvaluationContext.java | 2 +-
.../translation/PipelineTranslator.java | 2 +-
.../translation/SparkSessionFactory.java | 8 +-
.../translation/TransformTranslator.java | 2 +-
.../translation/batch/Aggregators.java | 5 +-
.../batch/CombineGloballyTranslatorBatch.java | 5 +-
.../batch/CombineGroupedValuesTranslatorBatch.java | 2 +-
.../batch/CombinePerKeyTranslatorBatch.java | 5 +-
.../batch/DoFnPartitionIteratorFactory.java | 16 +-
.../translation/batch/DoFnRunnerFactory.java | 38 +-
.../translation/batch/DoFnRunnerWithMetrics.java | 2 +-
.../translation/batch/FlattenTranslatorBatch.java | 2 +-
.../translation/batch/GroupByKeyHelpers.java | 5 +-
.../batch/GroupByKeyTranslatorBatch.java | 5 +-
.../translation/batch/ImpulseTranslatorBatch.java | 5 +-
.../translation/batch/ParDoTranslatorBatch.java | 2 +-
.../batch/ReadSourceTranslatorBatch.java | 2 +-
.../batch/ReshuffleTranslatorBatch.java | 2 +-
.../batch/WindowAssignTranslatorBatch.java | 5 +-
.../GroupAlsoByWindowViaOutputBufferFn.java | 31 +-
.../batch/functions/SideInputValues.java | 4 +-
.../translation/helpers/EncoderHelpers.java | 5 +-
.../translation/batch/AggregatorsTest.java | 7 +-
.../batch/functions/SideInputValuesTest.java | 9 +-
.../translation/helpers/EncoderHelpersTest.java | 5 +-
.../spark/coders/SparkRunnerKryoRegistrator.java | 3 +-
.../apache/beam/runners/spark/io/SourceRDD.java | 5 +-
.../runners/spark/io/SparkUnboundedSource.java | 7 +-
.../SparkGroupAlsoByWindowViaWindowSet.java | 29 +-
.../runners/spark/stateful/StateSpecFunctions.java | 9 +-
.../runners/spark/translation/BoundedDataset.java | 17 +-
.../spark/translation/DoFnRunnerWithMetrics.java | 2 +-
.../spark/translation/EvaluationContext.java | 5 +-
.../spark/translation/GroupCombineFunctions.java | 4 +-
.../GroupNonMergingWindowsFunctions.java | 15 +-
.../spark/translation/MultiDoFnFunction.java | 2 +-
.../ReifyTimestampsAndWindowsFunction.java | 5 +-
.../spark/translation/SideInputMetadata.java | 2 +-
.../spark/translation/SparkAssignWindowFn.java | 5 +-
.../SparkBatchPortablePipelineTranslator.java | 15 +-
.../runners/spark/translation/SparkCombineFn.java | 18 +-
.../SparkExecutableStageExtractionFunction.java | 2 +-
.../translation/SparkExecutableStageFunction.java | 4 +-
.../SparkGroupAlsoByWindowViaOutputBufferFn.java | 31 +-
.../spark/translation/SparkInputDataProcessor.java | 22 +-
.../spark/translation/SparkPCollectionView.java | 2 +-
.../SparkStreamingPortablePipelineTranslator.java | 19 +-
.../spark/translation/TransformTranslator.java | 27 +-
.../spark/translation/TranslationUtils.java | 7 +-
.../translation/streaming/ParDoStateUpdateFn.java | 11 +-
.../streaming/StatefulStreamingParDoEvaluator.java | 11 +-
.../streaming/StreamingTransformTranslator.java | 21 +-
.../spark/translation/streaming/TestDStream.java | 7 +-
.../translation/streaming/UnboundedDataset.java | 2 +-
.../streaming/WatermarkSyncedDStream.java | 2 +-
.../runners/spark/util/SparkSideInputReader.java | 2 +-
.../GroupNonMergingWindowsFunctionsTest.java | 13 +-
.../spark/translation/SparkCombineFnTest.java | 7 +-
.../SparkExecutableStageFunctionTest.java | 21 +-
.../translation/SparkInputDataProcessorTest.java | 21 +-
.../spark/translation/TransformTranslatorTest.java | 25 +-
.../ValueAndCoderLazySerializableTest.java | 13 +-
.../twister2/Twister2BatchTranslationContext.java | 2 +-
.../twister2/Twister2TranslationContext.java | 2 +-
.../wrappers/Twister2BoundedSource.java | 5 +-
.../translation/wrappers/Twister2EmptySource.java | 2 +-
.../batch/AssignWindowTranslatorBatch.java | 2 +-
.../translators/batch/FlattenTranslatorBatch.java | 2 +-
.../batch/GroupByKeyTranslatorBatch.java | 7 +-
.../translators/batch/ImpulseTranslatorBatch.java | 2 +-
.../batch/PCollectionViewTranslatorBatch.java | 11 +-
.../batch/ParDoMultiOutputTranslatorBatch.java | 2 +-
.../batch/ReadSourceTranslatorBatch.java | 2 +-
.../functions/AssignWindowsFunction.java | 5 +-
.../translators/functions/ByteToElemFunction.java | 4 +-
.../functions/ByteToWindowFunction.java | 4 +-
.../functions/ByteToWindowFunctionPrimitive.java | 9 +-
.../translators/functions/DoFnFunction.java | 5 +-
.../translators/functions/ElemToBytesFunction.java | 9 +-
.../functions/GroupByWindowFunction.java | 31 +-
.../translators/functions/ImpulseSource.java | 5 +-
.../translators/functions/MapToTupleFunction.java | 13 +-
.../translators/functions/OutputTagFilter.java | 2 +-
.../runners/twister2/utils/TranslationUtils.java | 5 +-
.../twister2/utils/Twister2AssignContext.java | 2 +-
.../twister2/utils/Twister2SideInputReader.java | 2 +-
scripts/ci/issue-report/package-lock.json | 480 +++++++++-----------
scripts/ci/issue-report/package.json | 4 +-
sdks/go.mod | 59 +--
sdks/go.sum | 136 +++---
sdks/go/pkg/beam/runners/dataflow/dataflow.go | 30 ++
.../prism/internal/engine/elementmanager.go | 29 +-
.../beam/runners/prism/internal/engine/timers.go | 17 +-
.../runners/prism/internal/jobservices/artifact.go | 2 +
.../runners/prism/internal/jobservices/server.go | 2 +-
sdks/go/pkg/beam/runners/prism/internal/stage.go | 4 +-
sdks/go/pkg/beam/util/gcsx/gcs.go | 13 +
sdks/go/pkg/beam/util/gcsx/gcs_test.go | 47 ++
sdks/java/container/build.gradle | 1 -
sdks/java/container/java8/build.gradle | 24 -
.../java/container/java8/java8-security.properties | 47 --
.../container/java8/option-java8-security.json | 9 -
.../org/apache/beam/sdk/coders/DequeCoder.java | 5 +-
.../beam/sdk/options/PortablePipelineOptions.java | 2 +-
.../org/apache/beam/sdk/transforms/Create.java | 7 +-
.../beam/sdk/transforms/DoFnOutputReceivers.java | 6 +-
.../reflect/ByteBuddyDoFnInvokerFactory.java | 97 ++--
...edFile.java => WindowedValueMultiReceiver.java} | 23 +-
.../{Weighted.java => WindowedValueReceiver.java} | 10 +-
.../sdk/util/construction/CoderTranslators.java | 21 +-
.../beam/sdk/util/construction/Environments.java | 25 +-
.../sdk/util/construction/ModelCoderRegistrar.java | 12 +-
.../util/construction/RunnerPCollectionView.java | 2 +-
.../apache/beam/sdk/values/PCollectionViews.java | 20 +-
.../org/apache/beam/sdk/values/WindowedValue.java | 60 +++
.../WindowedValues.java} | 168 +++----
.../sdk/fn/data/BeamFnDataInboundObserverTest.java | 17 +-
.../org/apache/beam/sdk/transforms/CreateTest.java | 9 +-
.../beam/sdk/transforms/RedistributeTest.java | 10 +-
.../apache/beam/sdk/transforms/ReshuffleTest.java | 10 +-
.../apache/beam/sdk/util/WindowedValueTest.java | 43 +-
.../util/construction/CoderTranslationTest.java | 6 +-
.../sdk/util/construction/EnvironmentsTest.java | 15 +-
.../sdk/util/construction/ModelCodersTest.java | 2 +-
.../euphoria/core/testkit/WindowingTest.java | 1 +
.../jmh/PrecombineGroupingTableBenchmark.java | 11 +-
.../fn/harness/jmh/ProcessBundleBenchmark.java | 4 +-
.../beam/fn/harness/AssignWindowsRunner.java | 5 +-
.../beam/fn/harness/BeamFnDataReadRunner.java | 2 +-
.../beam/fn/harness/BeamFnDataWriteRunner.java | 2 +-
.../org/apache/beam/fn/harness/CombineRunners.java | 4 +-
.../org/apache/beam/fn/harness/FlattenRunner.java | 2 +-
.../apache/beam/fn/harness/FnApiDoFnRunner.java | 83 ++--
.../org/apache/beam/fn/harness/MapFnRunners.java | 2 +-
.../beam/fn/harness/PTransformRunnerFactory.java | 2 +-
.../beam/fn/harness/PrecombineGroupingTable.java | 7 +-
.../SplittablePairWithRestrictionDoFnRunner.java | 5 +-
...littableSplitAndSizeRestrictionsDoFnRunner.java | 5 +-
.../beam/fn/harness/WindowedSplitResult.java | 2 +-
.../fn/harness/control/ProcessBundleHandler.java | 2 +-
.../harness/data/PCollectionConsumerRegistry.java | 7 +-
.../beam/fn/harness/debug/ElementSample.java | 2 +-
.../beam/fn/harness/debug/OutputSampler.java | 5 +-
.../beam/fn/harness/state/FnApiStateAccessor.java | 6 +-
.../beam/fn/harness/AssignWindowsRunnerTest.java | 35 +-
.../beam/fn/harness/BeamFnDataReadRunnerTest.java | 7 +-
.../beam/fn/harness/BeamFnDataWriteRunnerTest.java | 7 +-
.../apache/beam/fn/harness/CombineRunnersTest.java | 4 +-
.../apache/beam/fn/harness/FlattenRunnerTest.java | 9 +-
.../beam/fn/harness/FnApiDoFnRunnerTest.java | 95 ++--
.../apache/beam/fn/harness/MapFnRunnersTest.java | 13 +-
.../PTransformRunnerFactoryTestContext.java | 2 +-
.../fn/harness/PrecombineGroupingTableTest.java | 6 +-
...plittablePairWithRestrictionDoFnRunnerTest.java | 19 +-
...ableSplitAndSizeRestrictionsDoFnRunnerTest.java | 33 +-
.../fn/harness/data/BeamFnDataGrpcClientTest.java | 7 +-
.../data/PCollectionConsumerRegistryTest.java | 4 +-
.../beam/fn/harness/debug/DataSamplerTest.java | 5 +-
.../beam/fn/harness/debug/OutputSamplerTest.java | 41 +-
.../io/components/deadletterqueue/DLQRouter.java | 1 +
.../deadletterqueue/sinks/ThrowingSink.java | 2 +-
.../components/deadletterqueue/DLQRouterTest.java | 1 +
.../beam/sdk/io/gcp/bigquery/BigQueryUtils.java | 9 +-
.../sdk/io/gcp/bigquery/BigQueryHelpersTest.java | 4 +-
.../sdk/io/gcp/bigquery/BigQueryUtilsTest.java | 22 +-
sdks/java/io/iceberg/build.gradle | 3 +-
.../beam/sdk/io/iceberg/RecordWriterManager.java | 2 +-
.../sdk/io/iceberg/WriteGroupedRowsToFiles.java | 5 +-
.../sdk/io/iceberg/WriteUngroupedRowsToFiles.java | 5 +-
.../sdk/io/iceberg/RecordWriterManagerTest.java | 9 +-
.../java/org/apache/beam/sdk/io/jdbc/JdbcIO.java | 9 +-
.../io/jdbc/JdbcReadSchemaTransformProvider.java | 2 +-
.../jdbc/JdbcReadSchemaTransformProviderTest.java | 46 ++
sdks/java/testing/jpms-tests/build.gradle | 35 +-
README.md => sdks/python/README.md | 98 ++--
sdks/python/apache_beam/coders/coders.py | 9 +
.../apache_beam/coders/coders_test_common.py | 7 +
sdks/python/apache_beam/coders/row_coder.py | 3 +
.../apache_beam/examples/kafkataxi/README.md | 6 +-
.../internal/cloudpickle/cloudpickle.py | 74 ++--
.../apache_beam/internal/code_object_pickler.py | 46 ++
sdks/python/apache_beam/internal/dill_pickler.py | 7 +-
sdks/python/apache_beam/internal/pickler_test.py | 40 ++
sdks/python/apache_beam/io/filesystems.py | 2 +-
sdks/python/apache_beam/io/gcp/bigquery_test.py | 2 +-
sdks/python/apache_beam/io/gcp/pubsub_test.py | 3 +
sdks/python/apache_beam/io/localfilesystem.py | 4 +-
sdks/python/apache_beam/io/localfilesystem_test.py | 23 +-
sdks/python/apache_beam/ml/anomaly/transforms.py | 8 +-
.../apache_beam/ml/inference/vllm_inference.py | 1 +
sdks/python/apache_beam/pipeline.py | 24 +
sdks/python/apache_beam/pipeline_test.py | 38 ++
.../apache_beam/runners/dataflow/internal/names.py | 2 +-
.../Run Beam SQL with beam_sql magic.ipynb | 2 +-
.../portability/fn_api_runner/fn_runner_test.py | 106 +++++
.../runners/portability/portable_runner_test.py | 28 ++
.../external_transform_provider_it_test.py | 12 +
sdks/python/apache_beam/transforms/trigger_test.py | 27 +-
sdks/python/apache_beam/typehints/schemas.py | 4 +
sdks/python/apache_beam/yaml/README.md | 25 ++
.../{io/external/kafka.py => yaml/conftest.py} | 23 +-
.../python/apache_beam/yaml/examples}/__init__.py | 0
.../yaml/examples/testing/examples_test.py | 139 +-----
.../yaml/examples/testing/input_data.py | 146 ++++++
.../yaml/{tests => extended_tests}/bigquery.yaml | 0
.../yaml/{tests => extended_tests}/enrichment.yaml | 0
.../yaml/{tests => extended_tests}/jdbc.yaml | 0
.../yaml/{tests => extended_tests}/kafka.yaml | 0
.../yaml/{tests => extended_tests}/mysql.yaml | 0
.../yaml/{tests => extended_tests}/oracle.yaml | 0
.../yaml/{tests => extended_tests}/postgres.yaml | 20 +-
.../yaml/{tests => extended_tests}/pubsub.yaml | 0
.../yaml/{tests => extended_tests}/spanner.yaml | 0
.../yaml/{tests => extended_tests}/sqlserver.yaml | 0
.../yaml/{tests => extended_tests}/tfrecord.yaml | 0
sdks/python/apache_beam/yaml/generate_yaml_docs.py | 16 +-
sdks/python/apache_beam/yaml/integration_tests.py | 10 +-
sdks/python/apache_beam/yaml/yaml_provider.py | 2 +
sdks/python/apache_beam/yaml/yaml_specifiable.py | 59 +++
.../apache_beam/yaml/yaml_specifiable_test.py | 115 +++++
sdks/python/build.gradle | 21 +-
.../container/license_scripts/dep_urls_py.yaml | 2 +
sdks/python/gen_protos.py | 5 +-
sdks/python/setup.py | 33 +-
sdks/python/test-suites/dataflow/common.gradle | 2 +-
settings.gradle.kts | 1 -
.../www/site/content/en/case-studies/akvelon.md | 164 ++++++-
.../content/en/case-studies/creditKarmaInsights.md | 130 ++++++
.../documentation/pipelines/test-your-pipeline.md | 4 +-
.../en/documentation/runtime/environments.md | 7 +-
.../en/documentation/sdks/feature-comparison.md | 18 -
.../www/site/content/en/documentation/sdks/yaml.md | 46 ++
website/www/site/data/en/quotes.yaml | 10 +
website/www/site/layouts/partials/header.html | 6 -
.../beam-college/beam-college-25-desktop.png | Bin 230979 -> 0 bytes
.../beam-college/beam-college-25-mobile.png | Bin 113671 -> 0 bytes
.../static/images/case-study/akvelon/chitnis.png | Bin 0 -> 140126 bytes
.../images/case-study/akvelon/diagram-01.png | Bin 0 -> 105049 bytes
.../images/case-study/akvelon/diagram-02.png | Bin 0 -> 73850 bytes
.../static/images/case-study/akvelon/pikle.png | Bin 0 -> 502346 bytes
.../static/images/case-study/akvelon/terentyev.png | Bin 0 -> 815004 bytes
.../credit_karma_financial_insights.png | Bin 0 -> 138221 bytes
623 files changed, 5579 insertions(+), 3889 deletions(-)
copy .github/trigger_files/{beam_PostCommit_SQL.json =>
beam_PostCommit_Java_Examples_Dataflow_ARM.json} (100%)
create mode 100644
.github/trigger_files/beam_PostCommit_Java_Jpms_Dataflow_Java11.json
copy .github/trigger_files/{beam_PostCommit_TransformService_Direct.json =>
beam_PostCommit_Yaml_Xlang_Direct.json} (84%)
copy .github/workflows/{beam_PostCommit_Python_MongoDBIO_IT.yml =>
beam_PostCommit_Yaml_Xlang_Direct.yml} (84%)
create mode 100644 .test-infra/tools/stale_cleaner.py
create mode 100644 .test-infra/tools/test_stale_cleaner.py
rename examples/java/sql/src/main/java/org/apache/beam/examples/{sql =>
}/SchemaTransformExample.java (99%)
rename examples/java/sql/src/main/java/org/apache/beam/examples/{sql =>
}/SqlTransformExample.java (99%)
delete mode 100644
runners/core-java/src/main/java/org/apache/beam/runners/core/OutputWindowedValue.java
delete mode 100644 sdks/java/container/java8/build.gradle
delete mode 100644 sdks/java/container/java8/java8-security.properties
delete mode 100644 sdks/java/container/java8/option-java8-security.json
copy sdks/java/core/src/main/java/org/apache/beam/sdk/util/{ShardedFile.java
=> WindowedValueMultiReceiver.java} (61%)
copy sdks/java/core/src/main/java/org/apache/beam/sdk/util/{Weighted.java =>
WindowedValueReceiver.java} (74%)
create mode 100644
sdks/java/core/src/main/java/org/apache/beam/sdk/values/WindowedValue.java
rename
sdks/java/core/src/main/java/org/apache/beam/sdk/{util/WindowedValue.java =>
values/WindowedValues.java} (88%)
copy README.md => sdks/python/README.md (60%)
create mode 100644 sdks/python/apache_beam/internal/code_object_pickler.py
copy sdks/python/apache_beam/{io/external/kafka.py => yaml/conftest.py} (61%)
copy {examples/notebooks/notebook_test_scripts =>
sdks/python/apache_beam/yaml/examples}/__init__.py (100%)
create mode 100644 sdks/python/apache_beam/yaml/examples/testing/input_data.py
rename sdks/python/apache_beam/yaml/{tests => extended_tests}/bigquery.yaml
(100%)
rename sdks/python/apache_beam/yaml/{tests => extended_tests}/enrichment.yaml
(100%)
rename sdks/python/apache_beam/yaml/{tests => extended_tests}/jdbc.yaml (100%)
rename sdks/python/apache_beam/yaml/{tests => extended_tests}/kafka.yaml (100%)
rename sdks/python/apache_beam/yaml/{tests => extended_tests}/mysql.yaml (100%)
rename sdks/python/apache_beam/yaml/{tests => extended_tests}/oracle.yaml
(100%)
rename sdks/python/apache_beam/yaml/{tests => extended_tests}/postgres.yaml
(75%)
rename sdks/python/apache_beam/yaml/{tests => extended_tests}/pubsub.yaml
(100%)
rename sdks/python/apache_beam/yaml/{tests => extended_tests}/spanner.yaml
(100%)
rename sdks/python/apache_beam/yaml/{tests => extended_tests}/sqlserver.yaml
(100%)
rename sdks/python/apache_beam/yaml/{tests => extended_tests}/tfrecord.yaml
(100%)
create mode 100644 sdks/python/apache_beam/yaml/yaml_specifiable.py
create mode 100644 sdks/python/apache_beam/yaml/yaml_specifiable_test.py
create mode 100644
website/www/site/content/en/case-studies/creditKarmaInsights.md
delete mode 100644
website/www/site/content/en/documentation/sdks/feature-comparison.md
delete mode 100644
website/www/site/static/images/banners/beam-college/beam-college-25-desktop.png
delete mode 100644
website/www/site/static/images/banners/beam-college/beam-college-25-mobile.png
create mode 100644
website/www/site/static/images/case-study/akvelon/chitnis.png
create mode 100644
website/www/site/static/images/case-study/akvelon/diagram-01.png
create mode 100644
website/www/site/static/images/case-study/akvelon/diagram-02.png
create mode 100644 website/www/site/static/images/case-study/akvelon/pikle.png
create mode 100644
website/www/site/static/images/case-study/akvelon/terentyev.png
create mode 100644
website/www/site/static/images/case-study/credit_karma/credit_karma_financial_insights.png
