This is an automated email from the ASF dual-hosted git repository.
github-actions[bot] pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from ad39be0f58b Clarify service account key issue reports (#39894)
add 4d2c83d9b66 Bump github.com/fsouza/fake-gcs-server from 1.56.0 to
1.56.1 in /sdks (#39902)
add 351f4c7eac9 Bump actions/setup-java from 5 to 6 (#39903)
add cf27972eb10 Update CHANGES.md with new known issues (#39898)
add 01559cc9127 Added schema_update_options to Python BigQuery writes
(#39078)
add 34a00c7781f AddFiles: extract bounded async task plumbing and Parquet
footer reads (#39896)
add 8d5a5300fa3 Remove nullness suppression in trigger implementation
add 73e5ecc2ea8 Merge pull request #39807: Remove nullness suppression in
trigger implementation
add 493a18f8217 Bump docker/setup-buildx-action from 4.2.0 to 4.3.0
(#39829)
add d14b6483d5e Pin googleapis-common-protos<1.70 for py310-tensorflow-212
(#39904)
add d7557af6a55 [Spark 4] Add streaming dispatch seam and lifecycle hooks
(#39906)
No new revisions were added by this update.
Summary of changes:
.../IO_Iceberg_Integration_Tests.json | 2 +-
.github/workflows/beam_PostCommit_Go.yml | 2 +-
.../workflows/beam_PostCommit_Go_Dataflow_ARM.yml | 2 +-
.../beam_PostCommit_Java_Examples_Dataflow_ARM.yml | 2 +-
.../beam_PostCommit_XVR_GoUsingJava_Dataflow.yml | 2 +-
.../workflows/beam_PreCommit_CommunityMetrics.yml | 2 +-
.github/workflows/beam_PreCommit_PythonDocker.yml | 2 +-
.../workflows/beam_Publish_Beam_SDK_Snapshots.yml | 2 +-
.../workflows/beam_Publish_Python_VLLM_Image.yml | 2 +-
...beam_Python_ValidatesContainer_Dataflow_ARM.yml | 2 +-
.github/workflows/build_release_candidate.yml | 12 +-
.github/workflows/build_runner_image.yml | 2 +-
.github/workflows/code_completion_plugin_tests.yml | 2 +-
.../republish_released_docker_containers.yml | 4 +-
.github/workflows/typescript_tests.yml | 4 +-
CHANGES.md | 5 +
.../core/triggers/AfterAllStateMachine.java | 5 +-
.../AfterDelayFromFirstElementStateMachine.java | 5 +-
.../core/triggers/AfterEachStateMachine.java | 26 ++-
.../core/triggers/AfterFirstStateMachine.java | 5 +-
.../core/triggers/AfterWatermarkStateMachine.java | 12 +-
.../core/triggers/DefaultTriggerStateMachine.java | 3 -
.../triggers/ExecutableTriggerStateMachine.java | 21 +-
.../core/triggers/OrFinallyStateMachine.java | 5 +-
.../core/triggers/RepeatedlyStateMachine.java | 5 +-
.../runners/core/triggers/TriggerStateMachine.java | 17 +-
.../TriggerStateMachineContextFactory.java | 23 ++-
.../core/triggers/AfterEachStateMachineTest.java | 54 +++++
runners/spark/4/build.gradle | 8 +-
.../SparkStructuredStreamingPipelineOptions.java | 34 ++++
.../SparkStructuredStreamingPipelineResult.java | 16 +-
.../SparkStructuredStreamingRunner.java | 26 ++-
.../translation/EvaluationContext.java | 18 +-
.../translation/PipelineTranslator.java | 22 ++-
.../translation/PipelineTranslatorFactory.java | 40 ++--
.../translation/SparkSessionFactory.java | 6 +
sdks/go.mod | 4 +-
sdks/go.sum | 8 +-
.../org/apache/beam/sdk/io/iceberg/AddFiles.java | 103 +++-------
.../beam/sdk/io/iceberg/BoundedAsyncTasks.java | 113 +++++++++++
.../apache/beam/sdk/io/iceberg/ParquetFooters.java | 51 +++++
.../apache/beam/sdk/io/iceberg/AddFilesTest.java | 4 +-
.../beam/sdk/io/iceberg/BoundedAsyncTasksTest.java | 220 +++++++++++++++++++++
.../beam/sdk/io/iceberg/ParquetFootersTest.java | 113 +++++++++++
sdks/python/apache_beam/io/gcp/bigquery.py | 90 ++++++++-
sdks/python/apache_beam/io/gcp/bigquery_test.py | 144 +++++++++++++-
sdks/python/tox.ini | 1 +
47 files changed, 1035 insertions(+), 216 deletions(-)
copy
sdks/java/core/src/main/java/org/apache/beam/sdk/util/ExposedByteArrayInputStream.java
=>
runners/spark/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslatorFactory.java
(51%)
create mode 100644
sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/BoundedAsyncTasks.java
create mode 100644
sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/ParquetFooters.java
create mode 100644
sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/BoundedAsyncTasksTest.java
create mode 100644
sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/ParquetFootersTest.java
