This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from de76cfa16d3 Parse values returned from Dataflow API to BoundedTrieData
(#34738)
add adc40f93dce Remove breaking PDone change (#35224)
add 6c12a845536 Generic Postgres + Cloudsql postgres embeddings. (#35215)
add 660ee4ddece Allow only one thread at a time to start the VLLM server.
(#35234)
add 893beb1309e [IcebergIO] Create namespaces if needed (#35228)
add 8a0c08bf976 Support configuring flush_count and max_row_bytes of
WriteToBigTable (#34761)
add c9958d28114 Update CHANGES.md (#35242)
add 4b8ae104323 [Beam SQL] Implement Catalog and CatalogManager (#35223)
add 7f09169252a [IcebergIO] Fix conversion logic for arrays of structs and
maps of structs; fix output Schema resolution with column pruning (#35230)
add 478e4c7816e fix long_description when the md file cannot be found
(#35246)
add e6f0eac093f [IcebergIO] Create tables with a partition spec (#34966)
No new revisions were added by this update.
Summary of changes:
.../IO_Iceberg_Integration_Tests.json | 3 +-
.../beam_PostCommit_Python_Xlang_Gcp_Direct.json | 2 +-
.github/trigger_files/beam_PostCommit_SQL.json | 5 +-
.github/workflows/IO_Iceberg_Unit_Tests.yml | 1 +
CHANGES.md | 15 +-
.../schemaio-expansion-service/build.gradle | 2 +
.../extensions/sql/src/main/codegen/config.fmpp | 8 +
.../sql/src/main/codegen/includes/parserImpls.ftl | 119 ++++
.../apache/beam/sdk/extensions/sql/BeamSqlCli.java | 18 +-
.../beam/sdk/extensions/sql/SqlTransform.java | 10 +-
.../sdk/extensions/sql/impl/BeamCalciteSchema.java | 56 +-
.../sql/impl/BeamCalciteSchemaFactory.java | 26 +
.../beam/sdk/extensions/sql/impl/BeamSqlEnv.java | 53 +-
.../sdk/extensions/sql/impl/JdbcConnection.java | 10 +-
.../beam/sdk/extensions/sql/impl/JdbcDriver.java | 32 +-
.../sql/impl/parser/SqlCreateCatalog.java | 180 +++++
.../sql/impl/parser/SqlCreateExternalTable.java | 2 +-
.../extensions/sql/impl/parser/SqlDdlNodes.java | 10 +
.../extensions/sql/impl/parser/SqlDropCatalog.java | 112 +++
.../extensions/sql/impl/parser/SqlSetCatalog.java | 99 +++
.../sdk/extensions/sql/meta/catalog/Catalog.java | 28 +-
.../sql/meta/catalog/CatalogManager.java | 61 ++
.../sql/meta/catalog/CatalogRegistrar.java} | 16 +-
.../sql/meta/catalog/EmptyCatalogManager.java | 62 ++
.../sql/meta/catalog/InMemoryCatalog.java | 56 ++
.../sql/meta/catalog/InMemoryCatalogManager.java | 118 ++++
.../meta/catalog/InMemoryCatalogRegistrar.java} | 13 +-
.../extensions/sql/meta/catalog}/package-info.java | 4 +-
.../beam/sdk/extensions/sql/BeamSqlCliTest.java | 180 +++++
.../sdk/extensions/sql/PubsubToBigqueryIT.java | 4 +
.../apache/beam/sdk/io/iceberg/FilterUtils.java | 51 ++
.../org/apache/beam/sdk/io/iceberg/IcebergIO.java | 2 +-
.../beam/sdk/io/iceberg/IcebergScanConfig.java | 70 +-
.../sdk/io/iceberg/IcebergTableCreateConfig.java | 19 +-
.../apache/beam/sdk/io/iceberg/IcebergUtils.java | 93 ++-
.../IcebergWriteSchemaTransformProvider.java | 17 +
.../apache/beam/sdk/io/iceberg/PartitionUtils.java | 89 +++
.../io/iceberg/PortableIcebergDestinations.java | 14 +-
.../apache/beam/sdk/io/iceberg/ReadFromTasks.java | 5 +-
.../beam/sdk/io/iceberg/RecordWriterManager.java | 75 +-
.../apache/beam/sdk/io/iceberg/ScanTaskReader.java | 26 +-
.../apache/beam/sdk/io/iceberg/ScanTaskSource.java | 6 -
.../beam/sdk/io/iceberg/FilterUtilsTest.java | 19 +
.../beam/sdk/io/iceberg/IcebergIOReadTest.java | 14 +-
.../beam/sdk/io/iceberg/IcebergIOWriteTest.java | 36 +
.../beam/sdk/io/iceberg/IcebergUtilsTest.java | 136 ++++
.../IcebergWriteSchemaTransformProviderTest.java | 91 ++-
.../beam/sdk/io/iceberg/PartitionUtilsTest.java | 218 ++++++
.../sdk/io/iceberg/RecordWriterManagerTest.java | 34 +-
.../catalog/BigQueryMetastoreCatalogIT.java | 51 +-
.../sdk/io/iceberg/catalog/HadoopCatalogIT.java | 21 +-
.../beam/sdk/io/iceberg/catalog/HiveCatalogIT.java | 34 +-
.../io/iceberg/catalog/IcebergCatalogBaseIT.java | 233 ++++--
.../apache/beam/sdk/tpcds/BeamSqlEnvRunner.java | 14 +-
sdks/python/apache_beam/internal/gcp/auth_test.py | 1 +
sdks/python/apache_beam/io/gcp/bigtableio.py | 40 +-
sdks/python/apache_beam/io/gcp/bigtableio_test.py | 24 +-
.../apache_beam/io/gcp/datastore/v1new/helper.py | 2 +-
.../apache_beam/ml/gcp/recommendations_ai.py | 2 +-
.../apache_beam/ml/gcp/videointelligenceml.py | 3 +-
sdks/python/apache_beam/ml/gcp/visionml.py | 3 +-
.../apache_beam/ml/inference/vllm_inference.py | 38 +-
.../python/apache_beam/ml/rag/ingestion/alloydb.py | 783 ++-------------------
.../ml/rag/ingestion/alloydb_it_test.py | 343 +++++----
.../apache_beam/ml/rag/ingestion/cloudsql.py | 220 ++++++
.../ml/rag/ingestion/cloudsql_it_test.py | 223 ++++++
.../apache_beam/ml/rag/ingestion/jdbc_common.py | 78 ++
.../apache_beam/ml/rag/ingestion/postgres.py | 209 ++++++
.../ingestion/{alloydb.py => postgres_common.py} | 483 ++-----------
.../{alloydb_it_test.py => postgres_it_test.py} | 293 ++------
.../apache_beam/ml/rag/ingestion/test_utils.py | 105 +++
sdks/python/apache_beam/pipeline.py | 9 +-
sdks/python/apache_beam/pipeline_test.py | 4 +-
sdks/python/setup.py | 13 +-
.../site/content/en/documentation/io/managed-io.md | 23 +
75 files changed, 3673 insertions(+), 1899 deletions(-)
create mode 100644
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/SqlCreateCatalog.java
create mode 100644
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/SqlDropCatalog.java
create mode 100644
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/SqlSetCatalog.java
copy
examples/java/cdap/hubspot/src/main/java/org/apache/beam/examples/complete/cdap/hubspot/options/CdapHubspotSourceOptions.java
=>
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/catalog/Catalog.java
(56%)
create mode 100644
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/catalog/CatalogManager.java
copy
sdks/java/{core/src/main/java/org/apache/beam/sdk/function/ThrowingFunction.java
=>
extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/catalog/CatalogRegistrar.java}
(69%)
create mode 100644
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/catalog/EmptyCatalogManager.java
create mode 100644
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/catalog/InMemoryCatalog.java
create mode 100644
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/catalog/InMemoryCatalogManager.java
copy
sdks/java/{io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/schemas/AwsSchemaRegistrar.java
=>
extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/catalog/InMemoryCatalogRegistrar.java}
(70%)
copy sdks/java/{core/jmh/src/main/java/org/apache/beam/sdk/jmh/io =>
extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/catalog}/package-info.java
(91%)
create mode 100644
sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/PartitionUtils.java
create mode 100644
sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/PartitionUtilsTest.java
create mode 100644 sdks/python/apache_beam/ml/rag/ingestion/cloudsql.py
create mode 100644 sdks/python/apache_beam/ml/rag/ingestion/cloudsql_it_test.py
create mode 100644 sdks/python/apache_beam/ml/rag/ingestion/jdbc_common.py
create mode 100644 sdks/python/apache_beam/ml/rag/ingestion/postgres.py
copy sdks/python/apache_beam/ml/rag/ingestion/{alloydb.py =>
postgres_common.py} (56%)
copy sdks/python/apache_beam/ml/rag/ingestion/{alloydb_it_test.py =>
postgres_it_test.py} (76%)
create mode 100644 sdks/python/apache_beam/ml/rag/ingestion/test_utils.py
