This is an automated email from the ASF dual-hosted git repository.
github-actions[bot] pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from b4db8d2a11d Fix documentation typos across website, contributor docs,
and Go SDK docs (#40316)
add f976cf70bc1 Bump cloud.google.com/go/profiler from 0.6.0 to 0.7.0 in
/sdks (#40326)
add 0f5b3eebf47 Bump cloud.google.com/go/pubsub/v2 from 2.6.2 to 2.7.0 in
/sdks (#40329)
add b88e57e3437 [BEAM-20503] Replace inert nullness suppressions in
BigQueryIOPushDownIT (#40301)
add 4681e27c0bf Bump cloud.google.com/go/bigtable from 1.57.0 to 1.58.0 in
/sdks (#40328)
add 9454faca036 Bump nodemailer from 9.1.1 to 10.0.2 in
/scripts/ci/issue-report (#40335)
add 306e7d128d8 Clarify element processing timeout help text (#40312)
add 76a53714d2b Nullness improvements for simple core transforms (#40022)
add 9b3c14c9e5b Treat a 404 from BigQuery table and dataset deletes as
success (#39934)
add 5e5aaf3ce8f reduce LeaderBoardIT flakiness (#40241)
add d382b050463 [#20501] Remove nullness suppressions from sql extensions
module (#40317)
add f66e0ceea03 Fix overflow in LexicographicKeyRangeTracker (#40330)
add bb9d98aec5f Merge pull request #40323 from
reuvenlax/improve_test_tune_validates_runner
add c965c6eabeb Exposing evictWritersWhenFull in AvroIO (#40336)
add ea3c5dfb210 [Iceberg CDC sink] Integration tests (#40230)
add bffb545c2b3 Addfiles partition bounds fixes (#40252)
add f915665446e [Gemini] Add compatibility classes between apitools client
and modern bq client (#39998)
add 01b18a16867 Make ValidatesRunner faster: Improvements to file staging
(#40320)
add 2b35850258a [Python] Run Dataflow TFT benchmark tests on Python 3.11
to avoid protobuf resolution clash (#40333)
add 7df6f2fb627 Fix ML container publishing in push3XX tasks (#40343)
add fa34c16167c Make ValidatesRunner faster: Share and cache GCS staging
across TestDataflowRunner executions (#40321)
add 78c9e104a36 Bump nodemailer from 10.0.2 to 10.0.9 in
/scripts/ci/issue-report (#40349)
No new revisions were added by this update.
Summary of changes:
.../IO_Iceberg_Integration_Tests.json | 2 +-
.../IO_Iceberg_Integration_Tests_Dataflow.json | 3 +-
.github/trigger_files/beam_PostCommit_Python.json | 4 +-
CHANGES.md | 1 +
runners/google-cloud-dataflow-java/build.gradle | 31 +-
.../beam/runners/dataflow/TestDataflowRunner.java | 6 +
.../beam/runners/dataflow/util/GcsStager.java | 45 +
.../beam/runners/dataflow/util/PackageUtil.java | 4 +-
.../runners/dataflow/TestDataflowRunnerTest.java | 20 +
scripts/ci/issue-report/package-lock.json | 16 +-
scripts/ci/issue-report/package.json | 2 +-
sdks/go.mod | 8 +-
sdks/go.sum | 16 +-
.../main/java/org/apache/beam/sdk/io/FileIO.java | 15 +
.../org/apache/beam/sdk/transforms/Distinct.java | 7 +-
.../org/apache/beam/sdk/transforms/Latest.java | 52 +-
.../java/org/apache/beam/sdk/transforms/Reify.java | 6 +-
.../org/apache/beam/sdk/transforms/Sample.java | 6 +-
.../beam/sdk/util/construction/Environments.java | 114 +-
.../ClasspathScanningResourcesDetector.java | 64 +-
.../apache/beam/sdk/values/TimestampedValue.java | 3 +
.../apache/beam/sdk/transforms/GroupByKeyTest.java | 2 -
.../apache/beam/sdk/transforms/LatestFnTest.java | 6 +-
.../org/apache/beam/sdk/transforms/ParDoTest.java | 17 +-
.../sdk/util/construction/EnvironmentsTest.java | 53 +
.../apache/beam/sdk/extensions/avro/io/AvroIO.java | 14 +
.../provider/bigquery/BigQueryIOPushDownIT.java | 18 +-
.../beam/sdk/extensions/sql/impl/BeamSqlEnv.java | 2 +
.../sql/meta/provider/parquet/ParquetTable.java | 11 +-
.../sql/meta/provider/test/TestTableUtils.java | 2 +-
.../sdk/io/gcp/bigquery/BigQueryServicesImpl.java | 63 +-
.../io/gcp/bigquery/BigQueryServicesImplTest.java | 40 +
sdks/java/io/iceberg/build.gradle | 17 +
.../org/apache/beam/sdk/io/iceberg/AddFiles.java | 241 +++-
.../beam/sdk/io/iceberg/BoundAdjustment.java | 395 ++++++
.../apache/beam/sdk/io/iceberg/DryRunReport.java | 2 +-
.../beam/sdk/io/iceberg/NameMappingUtils.java | 12 +
.../beam/sdk/io/iceberg/ParquetFieldIds.java | 185 +++
.../beam/sdk/io/iceberg/AddFilesMetricsTest.java | 680 +++++++++
.../apache/beam/sdk/io/iceberg/AddFilesTest.java | 118 +-
.../beam/sdk/io/iceberg/BoundAdjustmentTest.java | 391 ++++++
.../beam/sdk/io/iceberg/NameMappingUtilsTest.java | 31 +
.../beam/sdk/io/iceberg/ParquetFieldIdsTest.java | 412 ++++++
.../beam/sdk/io/iceberg/ParquetTestFiles.java | 170 +++
...CatalogIT.java => HadoopCatalogCdcWriteIT.java} | 9 +-
...veCatalogIT.java => HiveCatalogCdcWriteIT.java} | 17 +-
.../io/iceberg/catalog/IcebergCdcWriteBaseIT.java | 905 ++++++++++++
...BLMSIT.java => LakehouseCatalogCdcWriteIT.java} | 34 +-
.../sdk/io/iceberg/cdc/sink/CdcSinkTestUtils.java | 30 +-
.../iceberg/cdc/sink/DeltaLakeToIcebergCdcIT.java | 346 +++++
.../examples/complete/game/leader_board_it_test.py | 25 +-
sdks/python/apache_beam/io/gcp/bigquery_compat.py | 1454 ++++++++++++++++++++
.../apache_beam/io/gcp/bigquery_compat_test.py | 796 +++++++++++
sdks/python/apache_beam/io/range_trackers.py | 2 +-
sdks/python/apache_beam/io/range_trackers_test.py | 7 +
.../python/apache_beam/options/pipeline_options.py | 13 +-
sdks/python/container/build.gradle | 2 +-
sdks/python/test-suites/gradle.properties | 4 +-
58 files changed, 6709 insertions(+), 242 deletions(-)
create mode 100644
sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/BoundAdjustment.java
create mode 100644
sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/ParquetFieldIds.java
create mode 100644
sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/AddFilesMetricsTest.java
create mode 100644
sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/BoundAdjustmentTest.java
create mode 100644
sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/ParquetFieldIdsTest.java
create mode 100644
sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/ParquetTestFiles.java
copy
sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/catalog/{HadoopCatalogIT.java
=> HadoopCatalogCdcWriteIT.java} (89%)
copy
sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/catalog/{HiveCatalogIT.java
=> HiveCatalogCdcWriteIT.java} (87%)
create mode 100644
sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/catalog/IcebergCdcWriteBaseIT.java
copy
sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/catalog/{RESTCatalogBLMSIT.java
=> LakehouseCatalogCdcWriteIT.java} (60%)
create mode 100644
sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/cdc/sink/DeltaLakeToIcebergCdcIT.java
create mode 100644 sdks/python/apache_beam/io/gcp/bigquery_compat.py
create mode 100644 sdks/python/apache_beam/io/gcp/bigquery_compat_test.py
