This is an automated email from the ASF dual-hosted git repository.
github-actions[bot] pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from a8e7ffab371 Fix race conditions, error recovery, and exit handlers in
job servers (#38423)
add 9a6734bbe27 Fix PreCommit Java PVR Prism Loopback workflow (#38431)
add 81830f22515 Bump cloud.google.com/go/profiler from 0.4.3 to 0.6.0 in
/sdks (#38436)
add 2e6ff74e11e Bump golang.org/x/sys from 0.43.0 to 0.44.0 in /sdks
(#38435)
add 8363afe825f Bump cloud.google.com/go/spanner from 1.88.0 to 1.91.0 in
/sdks (#38434)
add 5733cc8115d Fix flaky BigQuery file loads by safely handling
concurrent mkdirs (#38426)
add a86f2ec49c8 upgrade test containers and fix issues (#38438)
add 399d9d7ea6b Introduce ValueKind to Java and add to WindowedValue
(#38315)
add 13bbd5c4d67 Reduce number of layers for Beam container images (#38440)
add f1bbb637e31 Fix deadlock in AsyncWrapper reset_state() (#38427)
add f01b9ddb05e Make Beartype use the default behavior in
is_consistent_with() (#38275)
add 1d008bad91f [runners-spark] Add Spark 4 runner (#38255)
add a72b78125ef Fix thread leak for LOOPBACK workers in external worker
pool (#38432)
add fba639a05ff Bump urllib3 from 2.6.3 to 2.7.0 in
/sdks/python/container/py312 (#38452)
No new revisions were added by this update.
Summary of changes:
...ValidatesRunner_Spark4StructuredStreaming.json} | 3 +-
...on => beam_PreCommit_Java_Spark4_Versions.json} | 3 +-
.github/workflows/README.md | 1 +
..._ValidatesRunner_Spark4StructuredStreaming.yml} | 25 +-
CHANGES.md | 2 +-
gradle.properties | 2 +-
.../runners/core/LateDataDroppingDoFnRunner.java | 3 +-
...TimeBoundedSplittableProcessElementInvoker.java | 6 +-
.../core/SplittableParDoViaKeyedWorkItems.java | 3 +-
.../beam/runners/dataflow/BatchViewOverrides.java | 6 +
.../dataflow/worker/UngroupedWindmillReader.java | 10 +-
.../dataflow/worker/WindmillKeyedWorkItem.java | 14 +-
.../dataflow/worker/util/ValueInEmptyWindows.java | 6 +
.../dataflow/worker/WindmillKeyedWorkItemTest.java | 43 +++
runners/spark/4/README.md | 73 +++++
runners/spark/4/build.gradle | 53 ++++
.../1.20 => spark/4}/job-server/build.gradle | 8 +-
.../io/BoundedDatasetFactory.java | 13 +-
.../batch/CombinePerKeyTranslatorBatch.java | 6 +-
.../translation/batch/GroupByKeyHelpers.java | 8 +-
.../batch/GroupByKeyTranslatorBatch.java | 78 ++---
.../translation/helpers/EncoderFactory.java | 333 +++++++++++++++++++++
.../translation/helpers/EncoderHelpers.java | 45 ++-
.../translation/utils/ScalaInterop.java | 6 +-
.../translation/helpers/EncoderHelpersTest.java | 9 +-
runners/spark/spark_runner.gradle | 23 +-
.../spark/coders/SparkRunnerKryoRegistrator.java | 28 +-
.../SparkGroupAlsoByWindowViaWindowSet.java | 4 +-
.../SparkStructuredStreamingPipelineResult.java | 1 +
.../io/BoundedDatasetFactory.java | 2 +-
.../translation/EvaluationContext.java | 10 +-
.../batch/CombinePerKeyTranslatorBatch.java | 2 +-
.../translation/batch/GroupByKeyHelpers.java | 4 +-
.../batch/GroupByKeyTranslatorBatch.java | 2 +-
.../translation/helpers/EncoderHelpers.java | 21 +-
.../apache/beam/runners/spark/util/TimerUtils.java | 6 +
.../coders/SparkRunnerKryoRegistratorTest.java | 49 +++
.../translation/EvaluationContextTest.java | 84 ++++++
sdks/go.mod | 27 +-
sdks/go.sum | 62 ++--
sdks/go/container/Dockerfile | 10 +-
.../integration/internal/containers/containers.go | 3 +-
.../test/integration/io/spannerio/test_helpers.go | 3 +-
.../go/test/integration/io/xlang/jdbc/jdbc_test.go | 5 +-
sdks/java/container/Dockerfile | 31 +-
.../org/apache/beam/sdk/values/OutputBuilder.java | 2 +
.../beam/sdk/values/ValueInSingleWindow.java | 52 +++-
.../java/org/apache/beam/sdk/values/ValueKind.java | 28 +-
.../org/apache/beam/sdk/values/ValueKindUtil.java | 54 ++++
.../org/apache/beam/sdk/values/WindowedValue.java | 4 +
.../org/apache/beam/sdk/values/WindowedValues.java | 230 +++++++++++---
.../sdk/transforms/MetadataPropagationTest.java | 8 +-
.../apache/beam/sdk/util/WindowedValueTest.java | 5 +-
.../apache/beam/fn/harness/FnApiDoFnRunner.java | 3 +-
.../apache_beam/io/gcp/bigquery_file_loads.py | 8 +-
.../python/apache_beam/options/pipeline_options.py | 5 +
.../runners/portability/prism_runner_test.py | 62 ++++
.../apache_beam/runners/worker/worker_pool_main.py | 11 +
sdks/python/apache_beam/transforms/async_dofn.py | 29 +-
.../apache_beam/transforms/async_dofn_test.py | 35 +++
sdks/python/apache_beam/typehints/typehints.py | 12 +-
.../python/apache_beam/typehints/typehints_test.py | 27 ++
.../container/py312/base_image_requirements.txt | 2 +-
settings.gradle.kts | 2 +
64 files changed, 1464 insertions(+), 251 deletions(-)
copy .github/trigger_files/{beam_PostCommit_TransformService_Direct.json =>
beam_PostCommit_Java_ValidatesRunner_Spark4StructuredStreaming.json} (66%)
copy .github/trigger_files/{beam_PostCommit_TransformService_Direct.json =>
beam_PreCommit_Java_Spark4_Versions.json} (66%)
copy
.github/workflows/{beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.yml
=> beam_PostCommit_Java_ValidatesRunner_Spark4StructuredStreaming.yml} (81%)
create mode 100644 runners/spark/4/README.md
create mode 100644 runners/spark/4/build.gradle
copy runners/{flink/1.20 => spark/4}/job-server/build.gradle (81%)
copy runners/spark/{ =>
4}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/io/BoundedDatasetFactory.java
(95%)
copy runners/spark/{ =>
4}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CombinePerKeyTranslatorBatch.java
(97%)
copy runners/spark/{ =>
4}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/GroupByKeyHelpers.java
(92%)
copy runners/spark/{ =>
4}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/GroupByKeyTranslatorBatch.java
(85%)
create mode 100644
runners/spark/4/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/EncoderFactory.java
copy runners/spark/{ =>
4}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/EncoderHelpers.java
(93%)
copy runners/spark/{ =>
4}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/utils/ScalaInterop.java
(94%)
copy runners/spark/{ =>
4}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/EncoderHelpersTest.java
(97%)
create mode 100644
runners/spark/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/EvaluationContextTest.java
copy
it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datastream/DatastreamResourceManagerException.java
=> sdks/java/core/src/main/java/org/apache/beam/sdk/values/ValueKind.java (56%)
create mode 100644
sdks/java/core/src/main/java/org/apache/beam/sdk/values/ValueKindUtil.java
