This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from 93aa62c2900 Merge pull request #31058 from svetakvsundhar/broken_links
add 49da7a34d99 Bump golang.org/x/net from 0.22.0 to 0.23.0 in /sdks
(#31054)
add 516ac97659f fix examples on website (#31066)
add 1a5dc1c7fab Replace Source type for BigTable and Kafka stress tests
(#30999)
add 8dcbf967a1a [#30083][prism] Stabilize additional teststream cases.
(#31046)
add 2472695eb04 filter out outdated workflow runs (#31071)
add f8b81211049 Fix test failure due to locale sensitivity. (#31069)
add 718af0d7652 Don't use isolated builds when building an sdist for a
pipeline package supplied in --setup_file option. (#31070)
add 5a86b3fc922 Bump golang.org/x/net from 0.17.0 to 0.23.0 in
/learning/katas/go (#31050)
add 3dbbad98c80 Bump golang.org/x/net from 0.17.0 to 0.23.0 in
/.test-infra/mock-apis (#31051)
add a0dad088980 Bump golang.org/x/net from 0.22.0 to 0.24.0 in /sdks
(#31065)
add 37609ba70fa Managed Transform protos & translation; Iceberg
SchemaTransforms & translation (#30910)
No new revisions were added by this update.
Summary of changes:
.../beam_PostCommit_Python_Xlang_Gcp_Direct.json | 3 +-
.../beam_PostCommit_Python_Xlang_IO_Dataflow.json | 3 +-
.../beam_PreCommit_Xlang_Generated_Transforms.yml | 2 +-
.../base/v0.33.2/kafka-persistent.yaml | 14 +-
.../gke-internal-load-balanced/kustomization.yaml | 8 +-
.../github/github_runs_prefetcher/code/main.py | 6 +-
.test-infra/mock-apis/go.mod | 6 +-
.test-infra/mock-apis/go.sum | 12 +-
.../it/gcp/bigtable/BigtableResourceManager.java | 2 +-
.../apache/beam/it/gcp/bigtable/BigTableIOST.java | 41 ++--
.../java/org/apache/beam/it/kafka/KafkaIOST.java | 30 +--
learning/katas/go/go.mod | 2 +-
learning/katas/go/go.sum | 20 +-
.../model/pipeline/v1/external_transforms.proto | 4 +
sdks/go.mod | 6 +-
sdks/go.sum | 12 +-
.../prism/internal/engine/elementmanager.go | 26 ++-
.../runners/prism/internal/engine/engine_test.go | 5 +
.../runners/prism/internal/engine/teststream.go | 49 +++-
.../beam/runners/prism/internal/worker/bundle.go | 5 +-
sdks/go/test/integration/primitives/teststream.go | 74 ++++++
.../test/integration/primitives/teststream_test.go | 20 ++
sdks/go/test/integration/primitives/timers.go | 2 +-
.../java/org/apache/beam/sdk/schemas/Schema.java | 54 ++++-
.../apache/beam/sdk/schemas/SchemaRegistry.java | 23 ++
.../sdk/schemas/annotations/DefaultSchema.java | 18 ++
.../transforms/SchemaTransformTranslation.java | 79 +++++++
.../transforms/TypedSchemaTransformProvider.java | 35 ++-
.../apache/beam/sdk/schemas/utils/YamlUtils.java | 21 +-
.../util/construction/PTransformTranslation.java | 11 +
.../main/java/org/apache/beam/sdk/values/Row.java | 49 ++++
.../beam/sdk/schemas/SchemaRegistryTest.java | 49 ++++
.../org/apache/beam/sdk/schemas/SchemaTest.java | 100 +++++++++
.../TypedSchemaTransformProviderTest.java | 24 +-
.../org/apache/beam/sdk/util/YamlUtilsTest.java | 33 +++
.../java/org/apache/beam/sdk/values/RowTest.java | 90 ++++++++
.../sdk/expansion/service/ExpansionService.java | 41 +++-
.../org/apache/beam/fn/harness/CachesTest.java | 5 +-
sdks/java/io/expansion-service/build.gradle | 2 +
.../FileWriteSchemaTransformProviderTest.java | 32 +--
...ueryExportReadSchemaTransformConfiguration.java | 14 --
...FileLoadsWriteSchemaTransformConfiguration.java | 14 --
...QueryExportReadSchemaTransformProviderTest.java | 25 +--
...yFileLoadsWriteSchemaTransformProviderTest.java | 12 +-
.../io/gcp/bigquery/BigQueryIOTranslationTest.java | 2 +-
sdks/java/io/iceberg/build.gradle | 1 +
.../java/org/apache/beam/io/iceberg/IcebergIO.java | 93 --------
.../{ => sdk}/io/iceberg/AppendFilesToTables.java | 17 +-
.../{ => sdk}/io/iceberg/AssignDestinations.java | 2 +-
.../{ => sdk}/io/iceberg/DynamicDestinations.java | 2 +-
.../beam/{ => sdk}/io/iceberg/FileWriteResult.java | 12 +-
.../{ => sdk}/io/iceberg/IcebergCatalogConfig.java | 2 +-
.../{ => sdk}/io/iceberg/IcebergDestination.java | 2 +-
.../org/apache/beam/sdk/io/iceberg/IcebergIO.java | 136 +++++++++++
.../IcebergReadSchemaTransformProvider.java | 134 +++++++++++
.../{ => sdk}/io/iceberg/IcebergScanConfig.java | 2 +-
.../IcebergSchemaTransformCatalogConfig.java | 107 +++++++++
.../iceberg/IcebergSchemaTransformTranslation.java | 88 ++++++++
.../io/iceberg/IcebergTableCreateConfig.java | 2 +-
.../{ => sdk}/io/iceberg/IcebergWriteResult.java | 13 +-
.../IcebergWriteSchemaTransformProvider.java | 179 +++++++++++++++
.../io/iceberg/OneTableDynamicDestinations.java | 31 ++-
.../beam/{ => sdk}/io/iceberg/PropertyBuilder.java | 2 +-
.../beam/{ => sdk}/io/iceberg/RecordWriter.java | 4 +-
.../beam/{ => sdk}/io/iceberg/ScanSource.java | 2 +-
.../beam/{ => sdk}/io/iceberg/ScanTaskReader.java | 2 +-
.../beam/{ => sdk}/io/iceberg/ScanTaskSource.java | 2 +-
.../io/iceberg/SchemaAndRowConversions.java | 2 +-
.../apache/beam/sdk/io/iceberg/SnapshotInfo.java | 118 ++++++++++
.../io/iceberg/WriteGroupedRowsToFiles.java | 2 +-
.../{ => sdk}/io/iceberg/WriteToDestinations.java | 5 +-
.../io/iceberg/WriteUngroupedRowsToFiles.java | 2 +-
.../beam/{ => sdk}/io/iceberg/package-info.java | 2 +-
.../{ => sdk}/io/iceberg/FileWriteResultTest.java | 17 +-
.../{ => sdk}/io/iceberg/IcebergIOReadTest.java | 4 +-
.../{ => sdk}/io/iceberg/IcebergIOWriteTest.java | 16 +-
.../IcebergReadSchemaTransformProviderTest.java | 183 +++++++++++++++
.../IcebergSchemaTransformTranslationTest.java | 248 +++++++++++++++++++++
.../IcebergWriteSchemaTransformProviderTest.java | 175 +++++++++++++++
.../beam/{ => sdk}/io/iceberg/ScanSourceTest.java | 2 +-
.../io/iceberg/SchemaAndRowConversionsTest.java | 2 +-
.../{ => sdk}/io/iceberg/TestDataWarehouse.java | 2 +-
.../beam/{ => sdk}/io/iceberg/TestFixtures.java | 2 +-
.../KafkaReadSchemaTransformProviderTest.java | 16 +-
sdks/java/managed/build.gradle | 1 -
.../java/org/apache/beam/sdk/managed/Managed.java | 39 ++--
.../managed/ManagedSchemaTransformProvider.java | 119 ++++++----
.../managed/ManagedSchemaTransformTranslation.java | 59 +++++
.../sdk/managed/ManagedTransformConstants.java} | 9 +-
.../testing}/TestSchemaTransformProvider.java | 8 +-
.../beam/sdk/managed/testing}/package-info.java | 4 +-
.../ManagedSchemaTransformProviderTest.java | 54 ++++-
.../ManagedSchemaTransformTranslationTest.java | 185 +++++++++++++++
.../org/apache/beam/sdk/managed/ManagedTest.java | 1 +
sdks/python/apache_beam/io/gcp/bigquery.py | 14 +-
sdks/python/apache_beam/io/gcp/bigtableio.py | 12 +-
.../apache_beam/runners/portability/stager.py | 1 +
.../transforms/external_transform_provider.py | 35 +--
.../external_transform_provider_it_test.py | 22 --
sdks/python/apache_beam/yaml/yaml_provider.py | 2 +-
sdks/python/build.gradle | 2 +-
sdks/python/gen_xlang_wrappers.py | 21 +-
sdks/standard_expansion_services.yaml | 3 +
sdks/standard_external_transforms.yaml | 5 +-
.../content/en/documentation/runners/dataflow.md | 5 +-
.../site/content/en/get-started/quickstart-java.md | 2 +-
106 files changed, 2791 insertions(+), 530 deletions(-)
create mode 100644
sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/SchemaTransformTranslation.java
delete mode 100644
sdks/java/io/iceberg/src/main/java/org/apache/beam/io/iceberg/IcebergIO.java
rename sdks/java/io/iceberg/src/main/java/org/apache/beam/{ =>
sdk}/io/iceberg/AppendFilesToTables.java (87%)
rename sdks/java/io/iceberg/src/main/java/org/apache/beam/{ =>
sdk}/io/iceberg/AssignDestinations.java (98%)
rename sdks/java/io/iceberg/src/main/java/org/apache/beam/{ =>
sdk}/io/iceberg/DynamicDestinations.java (96%)
rename sdks/java/io/iceberg/src/main/java/org/apache/beam/{ =>
sdk}/io/iceberg/FileWriteResult.java (94%)
rename sdks/java/io/iceberg/src/main/java/org/apache/beam/{ =>
sdk}/io/iceberg/IcebergCatalogConfig.java (99%)
rename sdks/java/io/iceberg/src/main/java/org/apache/beam/{ =>
sdk}/io/iceberg/IcebergDestination.java (98%)
create mode 100644
sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java
create mode 100644
sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergReadSchemaTransformProvider.java
rename sdks/java/io/iceberg/src/main/java/org/apache/beam/{ =>
sdk}/io/iceberg/IcebergScanConfig.java (99%)
create mode 100644
sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergSchemaTransformCatalogConfig.java
create mode 100644
sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergSchemaTransformTranslation.java
rename sdks/java/io/iceberg/src/main/java/org/apache/beam/{ =>
sdk}/io/iceberg/IcebergTableCreateConfig.java (97%)
rename sdks/java/io/iceberg/src/main/java/org/apache/beam/{ =>
sdk}/io/iceberg/IcebergWriteResult.java (85%)
create mode 100644
sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergWriteSchemaTransformProvider.java
rename sdks/java/io/iceberg/src/main/java/org/apache/beam/{ =>
sdk}/io/iceberg/OneTableDynamicDestinations.java (66%)
rename sdks/java/io/iceberg/src/main/java/org/apache/beam/{ =>
sdk}/io/iceberg/PropertyBuilder.java (97%)
rename sdks/java/io/iceberg/src/main/java/org/apache/beam/{ =>
sdk}/io/iceberg/RecordWriter.java (96%)
rename sdks/java/io/iceberg/src/main/java/org/apache/beam/{ =>
sdk}/io/iceberg/ScanSource.java (99%)
rename sdks/java/io/iceberg/src/main/java/org/apache/beam/{ =>
sdk}/io/iceberg/ScanTaskReader.java (99%)
rename sdks/java/io/iceberg/src/main/java/org/apache/beam/{ =>
sdk}/io/iceberg/ScanTaskSource.java (98%)
rename sdks/java/io/iceberg/src/main/java/org/apache/beam/{ =>
sdk}/io/iceberg/SchemaAndRowConversions.java (99%)
create mode 100644
sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/SnapshotInfo.java
rename sdks/java/io/iceberg/src/main/java/org/apache/beam/{ =>
sdk}/io/iceberg/WriteGroupedRowsToFiles.java (98%)
rename sdks/java/io/iceberg/src/main/java/org/apache/beam/{ =>
sdk}/io/iceberg/WriteToDestinations.java (97%)
rename sdks/java/io/iceberg/src/main/java/org/apache/beam/{ =>
sdk}/io/iceberg/WriteUngroupedRowsToFiles.java (99%)
copy sdks/java/io/iceberg/src/main/java/org/apache/beam/{ =>
sdk}/io/iceberg/package-info.java (95%)
rename sdks/java/io/iceberg/src/test/java/org/apache/beam/{ =>
sdk}/io/iceberg/FileWriteResultTest.java (91%)
rename sdks/java/io/iceberg/src/test/java/org/apache/beam/{ =>
sdk}/io/iceberg/IcebergIOReadTest.java (97%)
rename sdks/java/io/iceberg/src/test/java/org/apache/beam/{ =>
sdk}/io/iceberg/IcebergIOWriteTest.java (95%)
create mode 100644
sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergReadSchemaTransformProviderTest.java
create mode 100644
sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergSchemaTransformTranslationTest.java
create mode 100644
sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergWriteSchemaTransformProviderTest.java
rename sdks/java/io/iceberg/src/test/java/org/apache/beam/{ =>
sdk}/io/iceberg/ScanSourceTest.java (99%)
rename sdks/java/io/iceberg/src/test/java/org/apache/beam/{ =>
sdk}/io/iceberg/SchemaAndRowConversionsTest.java (99%)
rename sdks/java/io/iceberg/src/test/java/org/apache/beam/{ =>
sdk}/io/iceberg/TestDataWarehouse.java (99%)
rename sdks/java/io/iceberg/src/test/java/org/apache/beam/{ =>
sdk}/io/iceberg/TestFixtures.java (99%)
create mode 100644
sdks/java/managed/src/main/java/org/apache/beam/sdk/managed/ManagedSchemaTransformTranslation.java
copy
sdks/java/{io/iceberg/src/main/java/org/apache/beam/io/iceberg/package-info.java
=>
managed/src/main/java/org/apache/beam/sdk/managed/ManagedTransformConstants.java}
(68%)
rename sdks/java/managed/src/{test/java/org/apache/beam/sdk/managed =>
main/java/org/apache/beam/sdk/managed/testing}/TestSchemaTransformProvider.java
(91%)
rename sdks/java/{io/iceberg/src/main/java/org/apache/beam/io/iceberg =>
managed/src/main/java/org/apache/beam/sdk/managed/testing}/package-info.java
(90%)
create mode 100644
sdks/java/managed/src/test/java/org/apache/beam/sdk/managed/ManagedSchemaTransformTranslationTest.java
