This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from a6de47572b9 [Go SDK] s3 filesystem: Fix nillable content length,
update deps. (#32111)
add 6bdf63a9d2d Bump cloud.google.com/go/spanner from 1.64.0 to 1.66.0 in
/sdks (#32126)
add af6bf8a1423 Bump golang.org/x/net from 0.27.0 to 0.28.0 in /sdks
(#32128)
add 27741fb6c9b Bump cloud.google.com/go/profiler from 0.4.0 to 0.4.1 in
/sdks (#32125)
add 6d96ae2580d Fix Lineage name breaking change (#32122)
add 01100a3b2fe Generate python dependencies (#32132)
add 82c3b36af70 Bump golang.org/x/oauth2 from 0.21.0 to 0.22.0 in /sdks
(#32129)
add 4b476832020 Bump github.com/fsouza/fake-gcs-server from 1.47.7 to
1.49.2 in /sdks (#32124)
add cae9148ee1d Bump up google-cloud-storage version to fix data
corruption issue (#32135)
add 1c0cfa1ccba Expose watermarkIdleDurationThreshold parameter to the
user in SolaceIO (#32109)
add 679e9d799e5 Upgrade Beam to use Cython 3.
add c825434965e Update base image requirements.
add 2b7e84239ea Remove now unneeded langauge level specifications.
add 1de0c4670ee Add no-except to time-critical function.
add 00de74030e2 Merge branch 'master' into cython3
add 49e98e59fd2 Merge pull request #32112 Upgrade Apache Beam to use
Cython 3.x.
add f73a6d1570a Create Beam YAML Join documentation (#31494)
add ec152e28355 [#32139] Fail pipelines with Stateful SDFs. (#32140)
add 17298b5572e [#32115] Fix timer support, support timer clears. (#32119)
add b21a84a4cd6 Managed Iceberg hive support and integration tests (#32052)
add fc5a71db5ca [CsvIO]: Implement CsvIOParse::withCustomRecordParsing
method (#32142)
add 488996913ff Add support for setting an HTTP read timeout for
BigQueryIO (#32118)
No new revisions were added by this update.
Summary of changes:
.../IO_Iceberg_Integration_Tests.json | 2 +-
.github/workflows/IO_Iceberg_Integration_Tests.yml | 2 +-
sdks/go.mod | 16 +-
sdks/go.sum | 62 ++---
.../prism/internal/engine/elementmanager.go | 22 +-
.../beam/runners/prism/internal/engine/timers.go | 115 +++++----
.../prism/internal/jobservices/management.go | 9 +
.../java/org/apache/beam/sdk/metrics/Lineage.java | 4 +-
.../org/apache/beam/sdk/io/csv/CsvIOParse.java | 28 ++-
.../beam/sdk/io/csv/CsvIOParseConfiguration.java | 12 +-
.../org/apache/beam/sdk/io/csv/CsvIOParseTest.java | 127 +++++++++-
.../beam/sdk/io/gcp/bigquery/BigQueryOptions.java | 9 +-
.../sdk/io/gcp/bigquery/BigQueryServicesImpl.java | 1 +
sdks/java/io/iceberg/build.gradle | 5 +
sdks/java/io/iceberg/hive/build.gradle | 80 ++++++
sdks/java/io/iceberg/hive/exec/build.gradle | 58 +++++
.../sdk/io/iceberg/hive/IcebergHiveCatalogIT.java | 280 +++++++++++++++++++++
.../hive/testutils/HiveMetastoreExtension.java | 68 +++++
.../io/iceberg/hive/testutils/ScriptRunner.java | 203 +++++++++++++++
.../iceberg/hive/testutils/TestHiveMetastore.java | 273 ++++++++++++++++++++
.../src/test/resources/hive-schema-3.1.0.derby.sql | 267 ++++++++++++++++++++
.../beam/sdk/io/iceberg/IcebergCatalogConfig.java | 37 ++-
.../IcebergReadSchemaTransformProvider.java | 56 +----
.../IcebergWriteSchemaTransformProvider.java | 58 +----
.../io/iceberg/SchemaTransformConfiguration.java | 69 +++++
.../beam/sdk/io/iceberg/IcebergIOReadTest.java | 16 +-
.../beam/sdk/io/iceberg/IcebergIOWriteTest.java | 40 ++-
.../IcebergReadSchemaTransformProviderTest.java | 4 +-
.../IcebergSchemaTransformTranslationTest.java | 6 +
.../IcebergWriteSchemaTransformProviderTest.java | 5 +-
.../apache/beam/sdk/io/iceberg/ScanSourceTest.java | 33 ++-
.../org/apache/beam/sdk/io/solace/SolaceIO.java | 28 ++-
.../sdk/io/solace/read/UnboundedSolaceReader.java | 4 +-
.../sdk/io/solace/read/UnboundedSolaceSource.java | 9 +
.../sdk/io/solace/read/WatermarkParameters.java | 26 +-
.../beam/sdk/io/solace/read/WatermarkPolicy.java | 10 +-
.../apache/beam/sdk/io/solace/SolaceIOTest.java | 3 +-
sdks/python/apache_beam/coders/coder_impl.py | 2 -
sdks/python/apache_beam/coders/stream.pyx | 2 -
sdks/python/apache_beam/metrics/cells.py | 2 -
sdks/python/apache_beam/metrics/execution.py | 2 -
.../python/apache_beam/metrics/monitoring_infos.py | 2 -
sdks/python/apache_beam/runners/common.py | 1 -
.../runners/portability/prism_runner_test.py | 32 +++
sdks/python/apache_beam/runners/worker/logger.py | 2 -
.../apache_beam/runners/worker/opcounters.py | 2 -
.../apache_beam/runners/worker/operations.py | 2 -
.../runners/worker/statesampler_fast.pyx | 4 +-
.../python/apache_beam/testing/fast_test_utils.pyx | 2 -
sdks/python/apache_beam/transforms/cy_combiners.py | 2 -
.../cy_dataflow_distribution_counter.pyx | 2 -
sdks/python/apache_beam/transforms/stats.py | 2 -
sdks/python/apache_beam/utils/counters.py | 1 -
sdks/python/apache_beam/utils/windowed_value.py | 2 -
.../container/base_image_requirements_manual.txt | 4 +-
.../container/license_scripts/dep_urls_py.yaml | 2 +
.../container/py310/base_image_requirements.txt | 137 +++++-----
.../container/py311/base_image_requirements.txt | 135 +++++-----
.../container/py312/base_image_requirements.txt | 124 ++++-----
.../container/py38/base_image_requirements.txt | 135 +++++-----
.../container/py39/base_image_requirements.txt | 141 +++++------
sdks/python/pyproject.toml | 2 +-
sdks/python/setup.py | 4 +-
settings.gradle.kts | 4 +
.../content/en/documentation/sdks/yaml-join.md | 182 ++++++++++++++
.../layouts/partials/section-menu/en/sdks.html | 1 +
66 files changed, 2325 insertions(+), 657 deletions(-)
create mode 100644 sdks/java/io/iceberg/hive/build.gradle
create mode 100644 sdks/java/io/iceberg/hive/exec/build.gradle
create mode 100644
sdks/java/io/iceberg/hive/src/test/java/org/apache/beam/sdk/io/iceberg/hive/IcebergHiveCatalogIT.java
create mode 100644
sdks/java/io/iceberg/hive/src/test/java/org/apache/beam/sdk/io/iceberg/hive/testutils/HiveMetastoreExtension.java
create mode 100644
sdks/java/io/iceberg/hive/src/test/java/org/apache/beam/sdk/io/iceberg/hive/testutils/ScriptRunner.java
create mode 100644
sdks/java/io/iceberg/hive/src/test/java/org/apache/beam/sdk/io/iceberg/hive/testutils/TestHiveMetastore.java
create mode 100644
sdks/java/io/iceberg/hive/src/test/resources/hive-schema-3.1.0.derby.sql
create mode 100644
sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/SchemaTransformConfiguration.java
create mode 100644 website/www/site/content/en/documentation/sdks/yaml-join.md
