This is an automated email from the ASF dual-hosted git repository.
ahmedabualsaud pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
from a0dad088980 Bump golang.org/x/net from 0.22.0 to 0.24.0 in /sdks
(#31065)
add 37609ba70fa Managed Transform protos & translation; Iceberg
SchemaTransforms & translation (#30910)
No new revisions were added by this update.
Summary of changes:
.../beam_PostCommit_Python_Xlang_Gcp_Direct.json | 3 +-
.../beam_PostCommit_Python_Xlang_IO_Dataflow.json | 3 +-
.../beam_PreCommit_Xlang_Generated_Transforms.yml | 2 +-
.../model/pipeline/v1/external_transforms.proto | 4 +
.../java/org/apache/beam/sdk/schemas/Schema.java | 54 ++++-
.../apache/beam/sdk/schemas/SchemaRegistry.java | 23 ++
.../sdk/schemas/annotations/DefaultSchema.java | 18 ++
.../transforms/SchemaTransformTranslation.java | 79 +++++++
.../transforms/TypedSchemaTransformProvider.java | 35 ++-
.../apache/beam/sdk/schemas/utils/YamlUtils.java | 21 +-
.../util/construction/PTransformTranslation.java | 11 +
.../main/java/org/apache/beam/sdk/values/Row.java | 49 ++++
.../beam/sdk/schemas/SchemaRegistryTest.java | 49 ++++
.../org/apache/beam/sdk/schemas/SchemaTest.java | 100 +++++++++
.../TypedSchemaTransformProviderTest.java | 24 +-
.../org/apache/beam/sdk/util/YamlUtilsTest.java | 33 +++
.../java/org/apache/beam/sdk/values/RowTest.java | 90 ++++++++
.../sdk/expansion/service/ExpansionService.java | 41 +++-
sdks/java/io/expansion-service/build.gradle | 2 +
.../FileWriteSchemaTransformProviderTest.java | 32 +--
...ueryExportReadSchemaTransformConfiguration.java | 14 --
...FileLoadsWriteSchemaTransformConfiguration.java | 14 --
...QueryExportReadSchemaTransformProviderTest.java | 25 +--
...yFileLoadsWriteSchemaTransformProviderTest.java | 12 +-
.../io/gcp/bigquery/BigQueryIOTranslationTest.java | 2 +-
sdks/java/io/iceberg/build.gradle | 1 +
.../java/org/apache/beam/io/iceberg/IcebergIO.java | 93 --------
.../{ => sdk}/io/iceberg/AppendFilesToTables.java | 17 +-
.../{ => sdk}/io/iceberg/AssignDestinations.java | 2 +-
.../{ => sdk}/io/iceberg/DynamicDestinations.java | 2 +-
.../beam/{ => sdk}/io/iceberg/FileWriteResult.java | 12 +-
.../{ => sdk}/io/iceberg/IcebergCatalogConfig.java | 2 +-
.../{ => sdk}/io/iceberg/IcebergDestination.java | 2 +-
.../org/apache/beam/sdk/io/iceberg/IcebergIO.java | 136 +++++++++++
.../IcebergReadSchemaTransformProvider.java | 134 +++++++++++
.../{ => sdk}/io/iceberg/IcebergScanConfig.java | 2 +-
.../IcebergSchemaTransformCatalogConfig.java | 107 +++++++++
.../iceberg/IcebergSchemaTransformTranslation.java | 88 ++++++++
.../io/iceberg/IcebergTableCreateConfig.java | 2 +-
.../{ => sdk}/io/iceberg/IcebergWriteResult.java | 13 +-
.../IcebergWriteSchemaTransformProvider.java | 179 +++++++++++++++
.../io/iceberg/OneTableDynamicDestinations.java | 31 ++-
.../beam/{ => sdk}/io/iceberg/PropertyBuilder.java | 2 +-
.../beam/{ => sdk}/io/iceberg/RecordWriter.java | 4 +-
.../beam/{ => sdk}/io/iceberg/ScanSource.java | 2 +-
.../beam/{ => sdk}/io/iceberg/ScanTaskReader.java | 2 +-
.../beam/{ => sdk}/io/iceberg/ScanTaskSource.java | 2 +-
.../io/iceberg/SchemaAndRowConversions.java | 2 +-
.../apache/beam/sdk/io/iceberg/SnapshotInfo.java | 118 ++++++++++
.../io/iceberg/WriteGroupedRowsToFiles.java | 2 +-
.../{ => sdk}/io/iceberg/WriteToDestinations.java | 5 +-
.../io/iceberg/WriteUngroupedRowsToFiles.java | 2 +-
.../beam/{ => sdk}/io/iceberg/package-info.java | 2 +-
.../{ => sdk}/io/iceberg/FileWriteResultTest.java | 17 +-
.../{ => sdk}/io/iceberg/IcebergIOReadTest.java | 4 +-
.../{ => sdk}/io/iceberg/IcebergIOWriteTest.java | 16 +-
.../IcebergReadSchemaTransformProviderTest.java | 183 +++++++++++++++
.../IcebergSchemaTransformTranslationTest.java | 248 +++++++++++++++++++++
.../IcebergWriteSchemaTransformProviderTest.java | 175 +++++++++++++++
.../beam/{ => sdk}/io/iceberg/ScanSourceTest.java | 2 +-
.../io/iceberg/SchemaAndRowConversionsTest.java | 2 +-
.../{ => sdk}/io/iceberg/TestDataWarehouse.java | 2 +-
.../beam/{ => sdk}/io/iceberg/TestFixtures.java | 2 +-
.../KafkaReadSchemaTransformProviderTest.java | 16 +-
sdks/java/managed/build.gradle | 1 -
.../java/org/apache/beam/sdk/managed/Managed.java | 39 ++--
.../managed/ManagedSchemaTransformProvider.java | 119 ++++++----
.../managed/ManagedSchemaTransformTranslation.java | 59 +++++
.../sdk/managed/ManagedTransformConstants.java} | 9 +-
.../testing}/TestSchemaTransformProvider.java | 8 +-
.../beam/sdk/managed/testing}/package-info.java | 4 +-
.../ManagedSchemaTransformProviderTest.java | 54 ++++-
.../ManagedSchemaTransformTranslationTest.java | 185 +++++++++++++++
.../org/apache/beam/sdk/managed/ManagedTest.java | 1 +
sdks/python/apache_beam/io/gcp/bigquery.py | 14 +-
sdks/python/apache_beam/io/gcp/bigtableio.py | 12 +-
.../transforms/external_transform_provider.py | 35 +--
.../external_transform_provider_it_test.py | 22 --
sdks/python/apache_beam/yaml/yaml_provider.py | 2 +-
sdks/python/build.gradle | 2 +-
sdks/python/gen_xlang_wrappers.py | 21 +-
sdks/standard_expansion_services.yaml | 3 +
sdks/standard_external_transforms.yaml | 5 +-
83 files changed, 2530 insertions(+), 438 deletions(-)
create mode 100644
sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/SchemaTransformTranslation.java
delete mode 100644
sdks/java/io/iceberg/src/main/java/org/apache/beam/io/iceberg/IcebergIO.java
rename sdks/java/io/iceberg/src/main/java/org/apache/beam/{ =>
sdk}/io/iceberg/AppendFilesToTables.java (87%)
rename sdks/java/io/iceberg/src/main/java/org/apache/beam/{ =>
sdk}/io/iceberg/AssignDestinations.java (98%)
rename sdks/java/io/iceberg/src/main/java/org/apache/beam/{ =>
sdk}/io/iceberg/DynamicDestinations.java (96%)
rename sdks/java/io/iceberg/src/main/java/org/apache/beam/{ =>
sdk}/io/iceberg/FileWriteResult.java (94%)
rename sdks/java/io/iceberg/src/main/java/org/apache/beam/{ =>
sdk}/io/iceberg/IcebergCatalogConfig.java (99%)
rename sdks/java/io/iceberg/src/main/java/org/apache/beam/{ =>
sdk}/io/iceberg/IcebergDestination.java (98%)
create mode 100644
sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java
create mode 100644
sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergReadSchemaTransformProvider.java
rename sdks/java/io/iceberg/src/main/java/org/apache/beam/{ =>
sdk}/io/iceberg/IcebergScanConfig.java (99%)
create mode 100644
sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergSchemaTransformCatalogConfig.java
create mode 100644
sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergSchemaTransformTranslation.java
rename sdks/java/io/iceberg/src/main/java/org/apache/beam/{ =>
sdk}/io/iceberg/IcebergTableCreateConfig.java (97%)
rename sdks/java/io/iceberg/src/main/java/org/apache/beam/{ =>
sdk}/io/iceberg/IcebergWriteResult.java (85%)
create mode 100644
sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergWriteSchemaTransformProvider.java
rename sdks/java/io/iceberg/src/main/java/org/apache/beam/{ =>
sdk}/io/iceberg/OneTableDynamicDestinations.java (66%)
rename sdks/java/io/iceberg/src/main/java/org/apache/beam/{ =>
sdk}/io/iceberg/PropertyBuilder.java (97%)
rename sdks/java/io/iceberg/src/main/java/org/apache/beam/{ =>
sdk}/io/iceberg/RecordWriter.java (96%)
rename sdks/java/io/iceberg/src/main/java/org/apache/beam/{ =>
sdk}/io/iceberg/ScanSource.java (99%)
rename sdks/java/io/iceberg/src/main/java/org/apache/beam/{ =>
sdk}/io/iceberg/ScanTaskReader.java (99%)
rename sdks/java/io/iceberg/src/main/java/org/apache/beam/{ =>
sdk}/io/iceberg/ScanTaskSource.java (98%)
rename sdks/java/io/iceberg/src/main/java/org/apache/beam/{ =>
sdk}/io/iceberg/SchemaAndRowConversions.java (99%)
create mode 100644
sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/SnapshotInfo.java
rename sdks/java/io/iceberg/src/main/java/org/apache/beam/{ =>
sdk}/io/iceberg/WriteGroupedRowsToFiles.java (98%)
rename sdks/java/io/iceberg/src/main/java/org/apache/beam/{ =>
sdk}/io/iceberg/WriteToDestinations.java (97%)
rename sdks/java/io/iceberg/src/main/java/org/apache/beam/{ =>
sdk}/io/iceberg/WriteUngroupedRowsToFiles.java (99%)
copy sdks/java/io/iceberg/src/main/java/org/apache/beam/{ =>
sdk}/io/iceberg/package-info.java (95%)
rename sdks/java/io/iceberg/src/test/java/org/apache/beam/{ =>
sdk}/io/iceberg/FileWriteResultTest.java (91%)
rename sdks/java/io/iceberg/src/test/java/org/apache/beam/{ =>
sdk}/io/iceberg/IcebergIOReadTest.java (97%)
rename sdks/java/io/iceberg/src/test/java/org/apache/beam/{ =>
sdk}/io/iceberg/IcebergIOWriteTest.java (95%)
create mode 100644
sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergReadSchemaTransformProviderTest.java
create mode 100644
sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergSchemaTransformTranslationTest.java
create mode 100644
sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergWriteSchemaTransformProviderTest.java
rename sdks/java/io/iceberg/src/test/java/org/apache/beam/{ =>
sdk}/io/iceberg/ScanSourceTest.java (99%)
rename sdks/java/io/iceberg/src/test/java/org/apache/beam/{ =>
sdk}/io/iceberg/SchemaAndRowConversionsTest.java (99%)
rename sdks/java/io/iceberg/src/test/java/org/apache/beam/{ =>
sdk}/io/iceberg/TestDataWarehouse.java (99%)
rename sdks/java/io/iceberg/src/test/java/org/apache/beam/{ =>
sdk}/io/iceberg/TestFixtures.java (99%)
create mode 100644
sdks/java/managed/src/main/java/org/apache/beam/sdk/managed/ManagedSchemaTransformTranslation.java
copy
sdks/java/{io/iceberg/src/main/java/org/apache/beam/io/iceberg/package-info.java
=>
managed/src/main/java/org/apache/beam/sdk/managed/ManagedTransformConstants.java}
(68%)
rename sdks/java/managed/src/{test/java/org/apache/beam/sdk/managed =>
main/java/org/apache/beam/sdk/managed/testing}/TestSchemaTransformProvider.java
(91%)
rename sdks/java/{io/iceberg/src/main/java/org/apache/beam/io/iceberg =>
managed/src/main/java/org/apache/beam/sdk/managed/testing}/package-info.java
(90%)
create mode 100644
sdks/java/managed/src/test/java/org/apache/beam/sdk/managed/ManagedSchemaTransformTranslationTest.java
