This is an automated email from the ASF dual-hosted git repository.
github-actions[bot] pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from 58bac320ebd [IcebergIO] Raise Java 17 floor for IcebergIO's Java 11
dependents (#39064)
add 2c735b67eb3 Bump cloud.google.com/go/spanner from 1.93.0 to 1.94.0 in
/sdks (#39550)
add c6e0f5b630e Bump github.com/aws/aws-sdk-go-v2/config in /sdks (#39551)
add 2fd6af992b5 Bump github.com/aws/aws-sdk-go-v2/service/s3 in /sdks
(#39554)
add 61fad4f672f Bump github.com/aws/aws-sdk-go-v2/feature/s3/manager in
/sdks (#39553)
add 525780e7af7 Provide a better error when beam plugin was supplied but
wasn't staged. (#39440)
add f65e0e0b0a8 Updates the Delta Lake source to support reading bounded
change data (#39426)
add 4f934835059 Fix module-level side effects and global random seeding in
univariate ML anomaly tests (#39462)
add bc991d99d17 Set envs
add 8a7c972c61b Merge pull request #39558 from apache/fix-auditkeys
add 621a78cd412 Add Beam YAML support for DebeziumIO
add 01c668b1455 Add Debezium YAML integration test
add a2267554a16 Fix record schema
add 25ed2f8b5b7 With max num of records
add aae48ec3ff0 Add setters
add 2541fc77cb3 Refactoring
add 315ed95a97d Fix python formatter
add b85646bf8df Remove primaryKeyColumns options
add 5178343b652 Fix spotless
add 0de9a676681 Merge pull request #39457 from apache/debezium-io-yaml
add b9e4e2f27a8 Bump docker/login-action from 4.5.2 to 4.6.0 (#39552)
add f7d8d7c8b58 Preserve partitioning on temp FILE_LOADS tables (#38833)
add 141804ab568 fix AddFilesIT filter for BigLake (#39533)
add f5feab8c598 Enhance Python Timestamp to be precision-variable up to
nanos, and map it to Timestamp logical type (#39537)
add 7b9380b1c25 Buffer BufferedLogger by newline to avoid log splitting
(#39288)
add 03db1a07096 Fix DataflowOutputCounter calculation for
ValueInEmptyWindows (#39487)
add b8d77b86055 [IcebergIO] Upgrade Iceberg dependency to 1.11.0 (#39559)
add 980c11432a1 Clean up legacy references to apitools in GCS I/O (#39433)
add 9c561e2983e [Iceberg] Make timestamptz return new Timestamp.MICROS
logical type (#39344)
No new revisions were added by this update.
Summary of changes:
.../IO_Iceberg_Integration_Tests.json | 2 +-
.../beam_PostCommit_Python_Xlang_Gcp_Direct.json | 2 +-
.../beam_PostCommit_Python_Xlang_IO_Direct.json | 2 +-
.../beam_Infrastructure_AuditUnmanagedKeys.yml | 5 +
.github/workflows/build_release_candidate.yml | 2 +-
.github/workflows/finalize_release.yml | 2 +-
CHANGES.md | 10 +-
.../beam/model/fnexecution/v1/standard_coders.yaml | 31 +
.../org/apache/beam/model/pipeline/v1/schema.proto | 13 +
.../core/GroupAlsoByWindowViaWindowSetNewDoFn.java | 2 +-
.../apache/beam/runners/core/KeyedWorkItem.java | 9 +
.../apache/beam/runners/core/ReduceFnRunner.java | 18 +-
.../dataflow/worker/DataflowOutputCounter.java | 68 +-
.../worker/IntrinsicMapTaskExecutorFactory.java | 11 +-
.../dataflow/worker/SimpleParDoFnHelpers.java | 5 +-
.../StreamingGroupAlsoByWindowViaWindowSetFn.java | 2 +-
.../dataflow/worker/WindmillKeyedWorkItem.java | 26 +-
.../dataflow/worker/DataflowOutputCounterTest.java | 108 +++
.../IntrinsicMapTaskExecutorFactoryTest.java | 14 +-
sdks/go.mod | 34 +-
sdks/go.sum | 68 +-
sdks/go/container/tools/buffered_logging.go | 64 +-
sdks/go/container/tools/buffered_logging_test.go | 168 +++-
.../apache/beam/sdk/schemas/SchemaTranslation.java | 2 +
.../beam/sdk/schemas/logicaltypes/Timestamp.java | 9 +-
.../beam/sdk/schemas/SchemaTranslationTest.java | 7 +
.../org/apache/beam/io/debezium/DebeziumIO.java | 16 +-
.../DebeziumReadSchemaTransformProvider.java | 96 ++-
.../DebeziumReadSchemaTransformProviderTest.java | 139 ++++
.../beam/sdk/io/delta/CreateCDCReadTasksDoFn.java | 296 ++++++++
.../apache/beam/sdk/io/delta/DeltaCDCReadTask.java | 125 +++
.../beam/sdk/io/delta/DeltaCDCSourceDoFn.java | 359 +++++++++
.../java/org/apache/beam/sdk/io/delta/DeltaIO.java | 150 +++-
.../io/delta/DeltaReadSchemaTransformProvider.java | 6 +-
.../apache/beam/sdk/io/delta/DeltaSourceDoFn.java | 2 +-
.../org/apache/beam/sdk/io/delta/DeltaIOTest.java | 841 +++++++++++++++++++--
.../beam/sdk/io/gcp/bigquery/BigQueryUtils.java | 6 +
.../sdk/io/gcp/bigquery/BigQueryUtilsTest.java | 42 +-
sdks/java/io/iceberg/build.gradle | 6 +-
.../org/apache/beam/sdk/io/iceberg/IcebergIO.java | 14 +-
.../beam/sdk/io/iceberg/IcebergScanConfig.java | 9 +-
.../apache/beam/sdk/io/iceberg/IcebergUtils.java | 76 +-
.../beam/sdk/io/iceberg/IncrementalScanSource.java | 4 +-
.../apache/beam/sdk/io/iceberg/ReadFromTasks.java | 4 +-
.../org/apache/beam/sdk/io/iceberg/ScanSource.java | 4 +-
.../apache/beam/sdk/io/iceberg/ScanTaskReader.java | 5 +-
.../org/apache/beam/sdk/io/iceberg/AddFilesIT.java | 15 +-
.../beam/sdk/io/iceberg/IcebergIOReadTest.java | 48 ++
.../beam/sdk/io/iceberg/IcebergUtilsTest.java | 43 +-
.../IcebergWriteSchemaTransformProviderTest.java | 13 +-
.../catalog/BigQueryMetastoreCatalogIT.java | 1 +
.../io/iceberg/catalog/IcebergCatalogBaseIT.java | 11 +-
sdks/python/apache_beam/coders/row_coder_test.py | 54 ++
sdks/python/apache_beam/io/filesystemio.py | 5 +-
sdks/python/apache_beam/io/gcp/__init__.py | 20 -
.../apache_beam/io/gcp/bigquery_file_loads.py | 83 +-
.../apache_beam/io/gcp/bigquery_file_loads_test.py | 151 ++++
.../apache_beam/io/gcp/bigquery_write_it_test.py | 71 +-
.../apache_beam/io/gcp/gcsfilesystem_test.py | 4 +-
.../apache_beam/ml/anomaly/univariate/mean_test.py | 8 +-
.../apache_beam/ml/anomaly/univariate/perf_test.py | 51 +-
.../ml/anomaly/univariate/quantile_test.py | 8 +-
.../ml/anomaly/univariate/stdev_test.py | 8 +-
sdks/python/apache_beam/portability/common_urns.py | 1 +
.../apache_beam/runners/worker/sdk_worker_main.py | 13 +-
.../transforms/managed_iceberg_it_test.py | 4 +-
sdks/python/apache_beam/typehints/schemas.py | 94 ++-
sdks/python/apache_beam/typehints/schemas_test.py | 94 +++
sdks/python/apache_beam/utils/timestamp.py | 327 ++++++--
sdks/python/apache_beam/utils/timestamp_test.py | 187 ++++-
.../databases/debezium.yaml} | 41 +-
sdks/python/apache_beam/yaml/integration_tests.py | 73 +-
sdks/python/apache_beam/yaml/standard_io.yaml | 23 +
sdks/python/build.gradle | 4 +-
sdks/python/container/boot.go | 23 +-
sdks/python/container/piputil.go | 39 +-
76 files changed, 3925 insertions(+), 476 deletions(-)
create mode 100644
runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/DataflowOutputCounterTest.java
create mode 100644
sdks/java/io/debezium/src/test/java/org/apache/beam/io/debezium/DebeziumReadSchemaTransformProviderTest.java
create mode 100644
sdks/java/io/delta/src/main/java/org/apache/beam/sdk/io/delta/CreateCDCReadTasksDoFn.java
create mode 100644
sdks/java/io/delta/src/main/java/org/apache/beam/sdk/io/delta/DeltaCDCReadTask.java
create mode 100644
sdks/java/io/delta/src/main/java/org/apache/beam/sdk/io/delta/DeltaCDCSourceDoFn.java
copy sdks/python/apache_beam/yaml/{tests/map.yaml =>
extended_tests/databases/debezium.yaml} (55%)
