This is an automated email from the ASF dual-hosted git repository.
github-actions[bot] pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from f7b52028e13 [Python] Fix ParquetIO silently corrupting data across
windows in unbounded pipelines (#40293)
add b3bda9ac42a fix messaging (#40392)
add 33547f6ccad more robust metrics (#40394)
add 0d374bf001e [Dataflow Streaming] [Multikey] Route only finalize
commits via multi key path (#40495)
add bb404806318 Update Debian
add 20897f39483 Merge pull request #40515 from apache/fix-playground-front
add d71149317eb [Python] Reset BigQuery retries for each batch (#39995)
add 61dd18e6886 set default otel.service.instance.id (#40353)
add ba0ba21e306 Bump golang.org/x/sys from 0.48.0 to 0.49.0 in /sdks
(#40523)
add c49f5c85846 Bump github/codeql-action from 4.38.2 to 4.38.3 (#40524)
add 091e6a84231 Bump google.golang.org/api from 0.300.0 to 0.301.0 in
/sdks (#40525)
add 3e915410508 Bump cloud.google.com/go/spanner from 1.95.1 to 1.96.1 in
/sdks (#40527)
add 3486966c4a1 [Python] Keep count/size labels in DataFrame agg with a
list (#40506)
add be5e581bc7e Bump golang.org/x/sync from 0.23.0 to 0.24.0 in /sdks
(#40526)
add 6ad41e6c3ea Bump github.com/aws/aws-sdk-go-v2/config from 1.33.6 to
1.33.8 in /sdks (#40522)
add b2b8f8bb3ae [Go SDK] Convert logical type arguments to and from schema
protos (#40153)
add 2a540512f76 Fix DaskRunner dropping finish_bundle outputs (#40519)
(#40520)
add d85fbc50cf9 [Python] Add `AwsOptions` and deprecate `S3Options`
(#40529)
add b5d160228fd Add mongodb_to_bigquery yaml blueprint example and e2e
test (#40485)
add 112739c3fea [SpannerIO] Add withWriteResults() to SpannerIO.Write to
emit commit metadata (#40480)
add 8d8b54954e3 Flink dropping watermarks hold fix (#40149)
add dbf9e500e56 [Python] Return every group from Regex.find_all with
Regex.ALL (#40399)
add 6a05766fd58 Fix OpenTelemetry dependency versions in generated
Dataflow runner Maven POM (#40530)
add 397136836cc [Go SDK] wrap a failed last data/timer send like Flush
(#40406)
add 0c47ade8d29 Adds an end-to-end example for migrating change data
(CDC)... (#40387)
add b96d6ba9e9a Clean up Python load test version configs (#40540)
add 3ec3db07aa1 [Python] Make `--aws_*` options defaults and `--s3_*`
options S3 overrides (#40532)
No new revisions were added by this update.
Summary of changes:
.github/workflows/codeql.yml | 4 +-
...aflow_MLTransform_Image_Embedding_CPU_Batch.txt | 1 -
...aflow_MLTransform_Image_Embedding_GPU_Batch.txt | 1 -
...s_Dataflow_MLTransform_Text_Embedding_Batch.txt | 1 -
...enchmarks_Dataflow_Pytorch_Image_Captioning.txt | 1 -
...rks_Dataflow_Pytorch_Image_Object_Detection.txt | 1 -
...nchmarks_Dataflow_Table_Row_Inference_Batch.txt | 1 -
...chmarks_Dataflow_Table_Row_Inference_Stream.txt | 1 -
...Python_Benchmarks_Dataflow_VLLM_Gemma_Batch.txt | 1 -
...python_CoGBK_Flink_Batch_100b_Multiple_Keys.txt | 1 -
.../python_CoGBK_Flink_Batch_100b_Single_Key.txt | 1 -
.../python_CoGBK_Flink_Batch_10kB.txt | 1 -
...bine_Dataflow_Streaming_2GB_10_byte_records.txt | 1 -
...hon_Combine_Dataflow_Streaming_2GB_Fanout_4.txt | 1 -
...hon_Combine_Dataflow_Streaming_2GB_Fanout_8.txt | 1 -
...hon_Combine_Flink_Batch_2GB_10_byte_records.txt | 1 -
.../python_Combine_Flink_Batch_2GB_Fanout_4.txt | 1 -
.../python_Combine_Flink_Batch_2GB_Fanout_8.txt | 1 -
...Combine_Flink_Streaming_2GB_10_byte_records.txt | 1 -
...python_Combine_Flink_Streaming_2GB_Fanout_4.txt | 1 -
...python_Combine_Flink_Streaming_2GB_Fanout_8.txt | 1 -
...thon_Combine_Flink_Streaming_small_Fanout_1.txt | 1 -
...thon_Combine_Flink_Streaming_small_Fanout_2.txt | 1 -
..._GBK_Dataflow_Streaming_2GB_of_100B_records.txt | 1 -
...GBK_Dataflow_Streaming_2GB_of_100kB_records.txt | 1 -
...n_GBK_Dataflow_Streaming_2GB_of_10B_records.txt | 1 -
...nout_4_times_with_2GB_10-byte_records_total.txt | 1 -
...nout_8_times_with_2GB_10-byte_records_total.txt | 1 -
.../python_GBK_Flink_Batch_2GB_of_100B_records.txt | 1 -
.../python_GBK_Flink_Batch_2GB_of_10B_records.txt | 1 -
...nout_4_times_with_2GB_10-byte_records_total.txt | 1 -
...nout_8_times_with_2GB_10-byte_records_total.txt | 1 -
...K_Flink_Batch_reiterate_4_times_10kB_values.txt | 1 -
...low_Streaming_reiterate_4_times_10kB_values.txt | 1 -
...flow_Streaming_reiterate_4_times_2MB_values.txt | 1 -
...ython_ParDo_Dataflow_Streaming_100_Counters.txt | 1 -
...python_ParDo_Dataflow_Streaming_10_Counters.txt | 1 -
...thon_ParDo_Dataflow_Streaming_10_Iterations.txt | 1 -
...hon_ParDo_Dataflow_Streaming_200_Iterations.txt | 1 -
.../python_ParDo_Flink_Batch_10_Counters.txt | 1 -
.../python_ParDo_Flink_Batch_10_Iterations.txt | 1 -
.../python_ParDo_Flink_Batch_200_Iterations.txt | 1 -
.../python_ParDo_Flink_Streaming_100_Counters.txt | 1 -
.../python_ParDo_Flink_Streaming_10_Counters.txt | 1 -
.../python_ParDo_Flink_Streaming_10_Iterations.txt | 1 -
...python_ParDo_Flink_Streaming_200_Iterations.txt | 1 -
.../python_ParDo_Flink_Streaming_5_Iterations.txt | 1 -
...taflow_Batch_10gb_1000window_first_iterable.txt | 1 -
...put_Dataflow_Batch_10gb_1000window_iterable.txt | 1 -
..._Dataflow_Batch_10gb_1window_first_iterable.txt | 1 -
...eInput_Dataflow_Batch_10gb_1window_iterable.txt | 1 -
...flow_Batch_1gb_1000window_1key_percent_dict.txt | 1 -
...low_Batch_1gb_1000window_99key_percent_dict.txt | 1 -
...ataflow_Batch_1gb_1window_1key_percent_dict.txt | 1 -
...taflow_Batch_1gb_1window_99key_percent_dict.txt | 1 -
...Input_Dataflow_Batch_1gb_1window_first_list.txt | 1 -
...n_SideInput_Dataflow_Batch_1gb_1window_list.txt | 1 -
CHANGES.md | 7 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 11 +-
examples/java/iceberg/build.gradle | 31 ++
.../iceberg/DeltaLakeToLakehouseCdcExample.java | 448 +++++++++++++++++++++
.../iceberg/DeltaLakeToLakehouseCdcUtils.java | 197 +++++++++
.../DeltaLakeToLakehouseCdcExampleTest.java | 96 +++++
.../iceberg/DeltaLakeToLakehouseCdcUtilsTest.java | 229 +++++++++++
playground/frontend/Dockerfile | 6 +-
.../streaming/stableinput/BufferingDoFnRunner.java | 24 +-
.../wrappers/streaming/DoFnOperatorTest.java | 91 +++++
runners/google-cloud-dataflow-java/build.gradle | 2 +
.../dataflow/worker/StreamingDataflowWorker.java | 18 +-
.../work/processing/StreamingWorkScheduler.java | 8 +-
.../worker/StreamingDataflowWorkerTest.java | 106 ++++-
sdks/go.mod | 42 +-
sdks/go.sum | 84 ++--
.../beam/core/runtime/graphx/schema/fieldvalue.go | 156 +++++++
.../core/runtime/graphx/schema/fieldvalue_test.go | 163 ++++++++
.../core/runtime/graphx/schema/logicaltypes.go | 71 +++-
.../pkg/beam/core/runtime/graphx/schema/schema.go | 165 ++++++--
.../beam/core/runtime/graphx/schema/schema_test.go | 334 +++++++++++++++
sdks/go/pkg/beam/core/runtime/harness/datamgr.go | 10 +-
.../pkg/beam/core/runtime/harness/datamgr_test.go | 38 ++
.../apache/beam/sdk/options/SdkHarnessOptions.java | 24 ++
.../java/org/apache/beam/fn/harness/FnHarness.java | 18 +-
.../apache/beam/sdk/io/gcp/spanner/SpannerIO.java | 156 ++++++-
.../sdk/io/gcp/spanner/SpannerWriteResult.java | 26 +-
.../beam/sdk/io/gcp/spanner/WriteResults.java | 122 ++++++
.../com/google/cloud/spanner/OptionsImposter.java | 8 +
.../sdk/io/gcp/spanner/SpannerIOWriteTest.java | 199 +++++++++
.../beam/sdk/io/gcp/spanner/SpannerWriteIT.java | 38 ++
.../sdk/io/iceberg/cdc/sink/CommitterMetrics.java | 73 ++--
.../sdk/io/iceberg/cdc/sink/OrderedCommitFn.java | 13 +-
.../beam/sdk/io/iceberg/cdc/sink/TableSetup.java | 21 +-
.../beam/sdk/io/iceberg/cdc/sink/WriteCdcRows.java | 3 +-
.../beam/sdk/io/iceberg/cdc/sink/package-info.java | 14 +-
.../sdk/io/iceberg/cdc/sink/CommitDeltasTest.java | 34 +-
.../io/iceberg/cdc/sink/CommitterMetricsTest.java | 27 +-
.../sdk/io/iceberg/cdc/sink/TableSetupTest.java | 2 +-
sdks/python/apache_beam/dataframe/frames.py | 6 +-
sdks/python/apache_beam/dataframe/frames_test.py | 8 +
.../apache_beam/io/aws/clients/s3/boto3_client.py | 42 +-
.../io/aws/clients/s3/boto3_client_test.py | 118 ++++++
.../apache_beam/io/aws/clients/s3/client_test.py | 2 +-
sdks/python/apache_beam/io/aws/s3filesystem.py | 2 +-
sdks/python/apache_beam/io/aws/s3io_test.py | 2 +-
sdks/python/apache_beam/io/gcp/bigquery.py | 14 +-
sdks/python/apache_beam/io/gcp/bigquery_test.py | 104 +++++
.../python/apache_beam/options/pipeline_options.py | 131 +++++-
.../apache_beam/options/pipeline_options_test.py | 96 +++++
.../apache_beam/runners/dask/dask_runner_test.py | 35 ++
.../runners/dask/transform_evaluator.py | 3 +-
.../apache_beam/testing/load_tests/build.gradle | 5 +
.../apache_beam/testing/synthetic_pipeline.py | 6 +-
sdks/python/apache_beam/transforms/util.py | 13 +-
sdks/python/apache_beam/transforms/util_test.py | 29 +-
.../yaml/examples/testing/examples_test.py | 45 ++-
...r_to_bigquery.yaml => mongodb_to_bigquery.yaml} | 39 +-
...l_to_bigquery.yaml => mongodb_to_bigquery.yaml} | 38 +-
116 files changed, 3496 insertions(+), 417 deletions(-)
create mode 100644
examples/java/iceberg/src/main/java/org/apache/beam/examples/iceberg/DeltaLakeToLakehouseCdcExample.java
create mode 100644
examples/java/iceberg/src/main/java/org/apache/beam/examples/iceberg/DeltaLakeToLakehouseCdcUtils.java
create mode 100644
examples/java/iceberg/src/test/java/org/apache/beam/examples/iceberg/DeltaLakeToLakehouseCdcExampleTest.java
create mode 100644
examples/java/iceberg/src/test/java/org/apache/beam/examples/iceberg/DeltaLakeToLakehouseCdcUtilsTest.java
create mode 100644 sdks/go/pkg/beam/core/runtime/graphx/schema/fieldvalue.go
create mode 100644
sdks/go/pkg/beam/core/runtime/graphx/schema/fieldvalue_test.go
create mode 100644
sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/WriteResults.java
create mode 100644
sdks/python/apache_beam/io/aws/clients/s3/boto3_client_test.py
copy
sdks/python/apache_beam/yaml/examples/transforms/blueprint/{spanner_to_bigquery.yaml
=> mongodb_to_bigquery.yaml} (68%)
copy
sdks/python/apache_beam/yaml/extended_tests/e2e/{cloudsql_to_bigquery.yaml =>
mongodb_to_bigquery.yaml} (71%)
