This is an automated email from the ASF dual-hosted git repository.
github-actions[bot] pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from 7b6413fd5ef Document Python BigQuery CDC record format (#40021)
add 2e7e9d371e8 Bump org.nosphere.apache.rat from 0.8.2 to 0.9.0 (#40097)
add 4581141552a Bump google-cloud-bigtable to 2.44.0 (#40100)
add 8b3370c310d AddFiles: SchemaDelta classifies what a file schema needs
from the table (#40062)
add f8221572fb2 [Spark][#36841] Translate stateless streaming pipelines on
the Spark 4 runner (#40090)
add 12126d8942a [CI] Stop infinite reviewer reassignment, batch state
commits, clean up stale state (#40096)
add 4621a0b229f Revert "[Experimental] Use zstd compression in Docker
builds" (#40105)
add 790468a6591 Ensure 0 backlog is sent when finishing processing
restrictions. (#40102)
No new revisions were added by this update.
Summary of changes:
...tCommit_Python_ValidatesContainer_Dataflow.json | 2 +-
.../beam_PreCommit_Flink_Container.json | 2 +-
build.gradle.kts | 2 +-
.../org/apache/beam/gradle/BeamDockerPlugin.groovy | 24 +-
...TimeBoundedSplittableProcessElementInvoker.java | 2 +-
.../core/SplittableParDoViaKeyedWorkItems.java | 3 +
...BoundedSplittableProcessElementInvokerTest.java | 13 +
.../runners/core/SplittableParDoProcessFnTest.java | 32 ++
.../translation/PipelineTranslatorFactory.java | 12 +-
.../translation/PipelineTranslatorStreaming.java | 113 ++++
.../translation/StreamingEvaluationContext.java | 261 +++++++++
.../streaming/ReadUnboundedTranslator.java | 97 ++++
.../io/streaming/BeamMicroBatchSourceTest.java | 269 +---------
.../io/streaming/TestUnboundedSource.java | 252 +++++++++
.../PipelineTranslatorStreamingTest.java | 142 +++++
.../streaming/StatelessParDoStreamingTest.java | 110 ++++
.../streaming/StreamingCheckpointRestartTest.java | 118 +++++
.../streaming/StreamingPipelineLifecycleTest.java | 171 ++++++
.../translation/streaming/StreamingTestUtils.java | 156 ++++++
.../translation/batch/ParDoTranslatorBatch.java | 2 +-
.../translation/batch/PipelineTranslatorBatch.java | 91 +---
...torBatch.java => PipelineTranslatorCommon.java} | 8 +-
.../batch/StatefulParDoTranslatorBatch.java | 2 +-
scripts/ci/pr-bot/README.md | 27 +
scripts/ci/pr-bot/findPrsNeedingAttention.ts | 61 ++-
scripts/ci/pr-bot/processNewPrs.ts | 31 +-
scripts/ci/pr-bot/shared/commentStrings.ts | 5 +-
scripts/ci/pr-bot/shared/constants.ts | 4 +
scripts/ci/pr-bot/shared/githubUtils.ts | 8 +
scripts/ci/pr-bot/shared/persistentState.ts | 59 ++-
scripts/ci/pr-bot/shared/pr.ts | 5 +
scripts/ci/pr-bot/shared/userCommand.ts | 4 +-
scripts/ci/pr-bot/test/githubUtilsTest.ts | 63 +++
scripts/ci/pr-bot/test/prTest.ts | 13 +
.../beam/sdk/io/iceberg/ColumnNameChecks.java | 154 ++++++
.../java/org/apache/beam/sdk/io/iceberg/Pins.java | 60 +++
.../apache/beam/sdk/io/iceberg/SchemaChange.java | 95 ++++
.../apache/beam/sdk/io/iceberg/SchemaDelta.java | 439 ++++++++++++++++
.../beam/sdk/io/iceberg/ColumnNameChecksTest.java | 258 +++++++++
.../beam/sdk/io/iceberg/SchemaChangeTest.java | 77 +++
.../beam/sdk/io/iceberg/SchemaDeltaTest.java | 582 +++++++++++++++++++++
.../container/ml/py310/base_image_requirements.txt | 2 +-
.../container/ml/py310/gpu_image_requirements.txt | 2 +-
.../container/ml/py311/base_image_requirements.txt | 2 +-
.../container/ml/py311/gpu_image_requirements.txt | 2 +-
.../container/ml/py312/base_image_requirements.txt | 2 +-
.../container/ml/py312/gpu_image_requirements.txt | 2 +-
.../container/ml/py313/base_image_requirements.txt | 2 +-
.../container/py310/base_image_requirements.txt | 2 +-
.../container/py311/base_image_requirements.txt | 2 +-
.../container/py312/base_image_requirements.txt | 2 +-
.../container/py313/base_image_requirements.txt | 2 +-
.../container/py314/base_image_requirements.txt | 2 +-
53 files changed, 3436 insertions(+), 417 deletions(-)
copy runners/spark/{ =>
4}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslatorFactory.java
(73%)
create mode 100644
runners/spark/4/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslatorStreaming.java
create mode 100644
runners/spark/4/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/StreamingEvaluationContext.java
create mode 100644
runners/spark/4/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/ReadUnboundedTranslator.java
create mode 100644
runners/spark/4/src/test/java/org/apache/beam/runners/spark/structuredstreaming/io/streaming/TestUnboundedSource.java
create mode 100644
runners/spark/4/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslatorStreamingTest.java
create mode 100644
runners/spark/4/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/StatelessParDoStreamingTest.java
create mode 100644
runners/spark/4/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/StreamingCheckpointRestartTest.java
create mode 100644
runners/spark/4/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/StreamingPipelineLifecycleTest.java
create mode 100644
runners/spark/4/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/StreamingTestUtils.java
copy
runners/spark/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/{PipelineTranslatorBatch.java
=> PipelineTranslatorCommon.java} (94%)
create mode 100644 scripts/ci/pr-bot/test/githubUtilsTest.ts
create mode 100644
sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/ColumnNameChecks.java
create mode 100644
sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/Pins.java
create mode 100644
sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/SchemaChange.java
create mode 100644
sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/SchemaDelta.java
create mode 100644
sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/ColumnNameChecksTest.java
create mode 100644
sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/SchemaChangeTest.java
create mode 100644
sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/SchemaDeltaTest.java
