This is an automated email from the ASF dual-hosted git repository.
damccorm pushed a change to branch users/damccorm-repro-35816
in repository https://gitbox.apache.org/repos/asf/beam.git
from 390fbfcd0c0 workflows: run python 3.12 only on precommit ml
`ubuntu-latest` os
add 5d7a8efb1b0 Revert "Add AftersynchronizedProcessing Time as
continuation trigger in Pytho…" (#35963)
add cffedbee518 Remove bad session test from SpannerReadIT (#36009)
add e67a5a26452 fix(mongodb): update filter composition to handle MongoDB
5.x driver (#36004)
add 4bcef0bd791 Update beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml
(#36015)
add 372b25b7571 Add GitHub Actions workflow for managing GCP Service
Account keys (#35911)
add dab5f5d7905 YAML example suite update (#35960)
add 258b8d1ed50 [Prism] Surface log format option and improve logging
presentation (#36008)
add 4bb6e5a11b5 python: replace <=3.12 with <3.13 for google-apitools
marker (semantics unchanged) (#35929)
add 6bdfcb3d30f update description and logs for timeout (#36003)
add e8fab2669a5 Pass update compat through as_deterministic_coder and use
cloudpickle for deterministic special types. (#35725)
add f5c53578a77 Add space character
add e72d9c7a41f fix pip extra options
add f1a0855b9bb Fix PostCommit Python ValidatesContainer Dataflow With RC
job (#36016)
add 17d5039a02f Bump github.com/aws/aws-sdk-go-v2/config from 1.31.4 to
1.31.6 in /sdks (#36025)
add 2ab70ffabea Upgrade Errorprone to 2.31.0 (#35684)
add 266de262f0e Bump cloud.google.com/go/bigquery from 1.69.0 to 1.70.0 in
/sdks (#36024)
add f69fd7230b4 Exit the poll loop in ReadFromKafkaDoFn after
consumerPollingTimeout has elapsed (#36029)
add a4ad5cd8b4e refactor(build): update changelog formatting in gradle
task (#36021)
add 9f3b160c100 Add validation to check for non-ints from partitionfns
(#35881)
add 523d833381a Cache deterministic types (#36032)
add f725e85e48f Update CHANGES.md (#36033)
add 4289ea20a8d Allow stateful exception handling (#35965)
add 7d6de9aa927 [Prism] Fix two kinds of errors when using json log
format. (#36020)
add 35969b34a37 Introduce more abstract handling of throttling signaling
(#35984)
add faacf7050e2 Bump cloud.google.com/go/spanner from 1.83.0 to 1.85.0 in
/sdks (#36005)
add f3441bd0c09 Moving to 2.69.0-SNAPSHOT on master branch.
add 6280b395220 Update CHANGES.md to have fields for 2.69.0 release
add 230f3ead708 Cloudpickle deterministic coder changes.md. (#36037)
add 9adc8dcd284 Support Protobuf 6.x. (#35477)
add cccaa75cc4d Only retrieve the shared caches once during setup in
ReadFromKafkaDoFn (#36028)
add d28fd643096 feat(build): add validation task for CHANGES.md formatting
(#36041)
add 67eaded97bf Fix jdbc logical type not found when python sdk worker
running in docker env (#36014)
add 0a536e7182d fix(Partition): support numpy integer types in partition
validation (#36044)
add 07ea0f8ae13 Bump github.com/aws/aws-sdk-go-v2/service/s3 in /sdks
(#36042)
add 20b52a12041 Comment send email action
add 19826274c62 Comment send email action (#36050)
add 32d53ab22e5 Update notebook to use embedding gemma (#36035)
add c367bd3347e Add Iceberg to AlloyDB Blueprint (#36036)
add 626be06a47b [YAML] Fraud Detection MLOps Workflow Example (#36012)
add 0c509d3a542 Add ALLOYDB_PASSWORD to Examples and Coverage workflow
(#36061)
add d2fdabb07e4 Update beam_PreCommit_Whitespace.yml to run
validateChanges (#36064)
add 6a7ef331dfe [Java] Fix expensive OneOfType precondition messages by
making them lazy (#36063)
add f10fbf0831d Add prism capability matrix (#36066)
add 9ac64e0bde2 Spanner: Extend SpannerIO support for experimental hosts
(#36048)
add d9ddcd4bf51 Add YAML Editor and Visualization Panel (#35947)
add 0ba89bbbb8f YAML example suite minor updates (#36074)
add 32f5be6c36c Update iceberg to 1.9.2 (#35981)
add a351c88d58c Added documentation for return vs yield usage in
DoFn.process() (#34912)
add ed383fa4a3b Logs the BigQuery error that caused the load job failure
(resolves #1… (#36068)
add 56dac7574cc Fix MqttIO read checkpoint logic (#36056)
add 56607ec888b Improve Iceberg Blueprint and documentation and Tests
(#35961)
add be002c7a23d [YAML]: add optional schema config for all transforms
(#35952)
add be4fb977e18 Support managed jdbc io (Postgres) (#36034)
add 32aff26a95b [yaml] - add yaml expand_pipeline ut tests (#36087)
add fe188e3635b [#34236] Add Vertex AI Multi-Modal embedding handler
(#35677)
add a2f9fb8e992 pin avro 1.12.0 in expansion service (#36092)
add dffa53f7624 Correctly exclude prism when using external transforms
(#36049)
add 28ad2358097 Add interactive dependency to notebooks (#36091)
add 10f0f072834 workflows: add `ALLOYDB_PASSWORD` to Precommit Python
Transforms (#36096)
add 6a4ce21a742 Use unsigned integer math to calculate progress in
GrowableOffsetRangeTracker since the difference between two offsets does not
exceed the maximum value of the unsigned representation.
add 48b453a6b01 Merge pull request #35561: [GrowableOffsetRangeTracker]
Use UnsignedLong instead of BigDecimal to calculate progress
add 520b456b769 Only update size metrics once per batch
add 60630afb39b Merge pull request #36077: [KafkaIO] Only update size
metrics once per batch
add 359c9ebec95 Introduce Schema Registry Functionality to Managed KafkaIO
Write. (#35644)
add 69aa8af121e Fix portableLocalRunnerJuliaSetWithSetupPy
add 246d95587e7 Fix PostCommit Python job (#36098)
add 9651c641240 Fix JDBC type validation to check for empty strings in
Read and Write Postgres schema transform providers (#36103)
add a2e09096964 Bump github.com/spf13/cobra from 1.9.1 to 1.10.1 in /sdks
(#36038)
add 33d8d702575 [YAML] - update readme to correct pipeline calls (#36076)
add 1da37bcff00 [2/2] sdks/python: enrich data with CloudSQL [PostgreSQL,
MySQL, SQLServer] (#35473)
add becfcf86ab6 Fix CVE-2025-48734, CVE-2024-13009 (#36106)
add e64815d106a Hint Suggestions for invalid pipeline options (#36072)
add 4f0bcf63a37 Pin Avro 1.12 in expansion service container (#36108)
add dbae047c6ad feat(pubsub): support batch mode in WriteToPubSub
transform (#36027)
add f96128a33bd Update PubsubIO.java documentation (#36120)
add 2a5bbc2d3b0 [Dataflow Java] Support the existing --logMdc option by
adding the present mdc values to custom_data json field.
add f359ff57b9c Merge pull request #36117: [Dataflow Java] Support the
existing --logMdc option by adding the present mdc values to custom_data json
field.
add 2beb75c5bf9 infra/security: update project_id and bucket_name to match
the correct configuration (#36122)
add abbaab8bfcd Fix flaky
GrpcDirectGetWorkStreamTest.testConsumedWorkItems_itemsSplitAcrossResponses
(#36129)
add ab2db45f71d Update import gpg action (#36130)
add f296000bb30 Update action (#36131)
new fb62994f8ec Merge branch 'master' of https://github.com/apache/beam
into users/damccorm-repro-35816
new ebee31449ec test-suites/direct/common.gradle: remove milvus postcommit
gradle hook
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.asf.yaml | 1 +
.../IO_Iceberg_Integration_Tests.json | 2 +-
.../IO_Iceberg_Integration_Tests_Dataflow.json | 2 +-
...Iceberg_Managed_Integration_Tests_Dataflow.json | 2 +-
...PostCommit_Python_ValidatesRunner_Dataflow.json | 3 +-
.../beam_PostCommit_Python_Xlang_Gcp_Dataflow.json | 2 +-
.../beam_PostCommit_Python_Xlang_IO_Direct.json | 2 +-
.github/trigger_files/beam_PostCommit_SQL.json | 2 +-
.../beam_IODatastoresCredentialsRotation.yml | 28 +-
... => beam_Infrastructure_ServiceAccountKeys.yml} | 46 +-
.../workflows/beam_MetricsCredentialsRotation.yml | 28 +-
.github/workflows/beam_Metrics_Report.yml | 26 +-
.../beam_PostCommit_Java_BigQueryEarlyRollout.yml | 28 +-
.../beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml | 4 +-
.../workflows/beam_PreCommit_Python_Examples.yml | 3 +-
.../workflows/beam_PreCommit_Python_Transforms.yml | 3 +-
.github/workflows/beam_PreCommit_Whitespace.yml | 5 +
.github/workflows/build_release_candidate.yml | 8 +-
CHANGES.md | 192 +--
build.gradle.kts | 251 ++++
buildSrc/build.gradle.kts | 2 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 141 ++-
contributor-docs/code-change-guide.md | 13 +
examples/java/build.gradle | 3 -
examples/java/{sql => iceberg}/build.gradle | 77 +-
.../iceberg}/IcebergBatchWriteExample.java | 2 +-
.../iceberg}/IcebergRestCatalogCDCExample.java | 2 +-
.../IcebergRestCatalogStreamingWriteExample.java | 2 +-
.../examples/iceberg}/IcebergTaxiExamples.java | 2 +-
.../examples/complete/StreamingWordExtract.java | 12 +-
.../datatokenization/utils/SchemasUtils.java | 2 +-
.../alloydb_product_catalog_embeddings.ipynb | 2 +-
.../anomaly_detection_iforest.ipynb | 2 +-
.../anomaly_detection_timesfm.ipynb | 2 +-
.../beam-ml/automatic_model_refresh.ipynb | 2 +-
.../bigquery_vector_ingestion_and_search.ipynb | 2 +-
...cloudsql_mysql_product_catalog_embeddings.ipynb | 2 +-
...udsql_postgres_product_catalog_embeddings.ipynb | 2 +-
.../compute_and_apply_vocab.ipynb | 2 +-
.../huggingface_text_embeddings.ipynb | 32 +-
.../beam-ml/data_preprocessing/scale_data.ipynb | 2 +-
.../vertex_ai_text_embeddings.ipynb | 2 +-
.../gemma_2_sentiment_and_summarization.ipynb | 4 +-
.../beam-ml/image_processing_tensorflow.ipynb | 3 +-
examples/notebooks/beam-ml/mltransform_basic.ipynb | 2 +-
examples/notebooks/beam-ml/per_key_models.ipynb | 2 +-
.../beam-ml/rag_usecase/beam_rag_notebook.ipynb | 2 +-
.../rag_usecase/opensearch_rag_pipeline.ipynb | 2 +-
.../notebooks/beam-ml/run_inference_gemma.ipynb | 2 +-
.../beam-ml/run_inference_generative_ai.ipynb | 2 +-
.../beam-ml/run_inference_multi_model.ipynb | 2 +-
.../notebooks/beam-ml/run_inference_pytorch.ipynb | 2 +-
.../run_inference_pytorch_tensorflow_sklearn.ipynb | 6 +-
.../beam-ml/run_inference_tensorflow.ipynb | 2 +-
.../run_inference_tensorflow_with_tfx.ipynb | 2 +-
.../beam-ml/run_inference_vertex_ai.ipynb | 2 +-
.../run_inference_with_tensorflow_hub.ipynb | 2 +-
.../beam-ml/speech_emotion_tensorflow.ipynb | 2 +-
examples/notebooks/blog/unittests_in_beam.ipynb | 2 +-
examples/yaml/README.md | 54 +
gradle.properties | 4 +-
infra/keys/README.md | 14 +-
infra/keys/config.yaml | 4 -
infra/keys/gcp_logger.py | 153 ---
infra/keys/keys.py | 101 +-
infra/keys/keys.yaml | 2 +-
infra/security/config.yml | 4 +-
.../beam/it/common/utils/ResourceManagerUtils.java | 3 +-
.../org/apache/beam/it/gcp/IOLoadTestBase.java | 2 +-
.../gcp/artifacts/matchers/ArtifactsSubject.java | 1 +
.../beam/it/gcp/bigquery/BigQueryStreamingLT.java | 4 +-
.../model/pipeline/v1/external_transforms.proto | 4 +
.../apache/beam/runners/core/PaneInfoTracker.java | 2 +-
.../core/SplittableParDoViaKeyedWorkItems.java | 3 +-
.../org/apache/beam/runners/core/StateMerging.java | 8 +-
.../org/apache/beam/runners/core/StateTags.java | 3 +-
.../apache/beam/runners/core/TimerInternals.java | 7 +-
.../apache/beam/runners/core/WatermarkHold.java | 7 +-
.../runners/core/metrics/MetricsContainerImpl.java | 5 +-
.../runners/core/metrics/SimpleExecutionState.java | 2 +-
.../triggers/ExecutableTriggerStateMachine.java | 8 +-
.../core/metrics/MetricsContainerStepMapTest.java | 2 +-
.../runners/core/metrics/MetricsPusherTest.java | 2 +-
.../triggers/ReshuffleTriggerStateMachineTest.java | 3 +-
.../beam/runners/direct/DirectRunnerTest.java | 2 +
.../runners/flink/FlinkDetachedRunnerResult.java | 2 +-
.../beam/runners/dataflow/BatchViewOverrides.java | 6 +-
.../beam/runners/dataflow/DataflowPipelineJob.java | 3 +-
.../google-cloud-dataflow-java/worker/build.gradle | 6 +-
.../dataflow/worker/WindmillTimerInternals.java | 70 +-
.../dataflow/worker/WorkerCustomSources.java | 3 +-
.../logging/DataflowWorkerLoggingHandler.java | 30 +
.../logging/DataflowWorkerLoggingInitializer.java | 4 +
.../worker/windmill/state/ToIterableFunction.java | 6 +-
.../dataflow/worker/GroupingShuffleReaderTest.java | 2 +-
.../dataflow/worker/IsmSideInputReaderTest.java | 6 +-
.../worker/StreamingSideInputDoFnRunnerTest.java | 6 +-
.../worker/StreamingSideInputFetcherTest.java | 2 +-
.../logging/DataflowWorkerLoggingHandlerTest.java | 61 +-
.../worker/util/GroupAlsoByWindowProperties.java | 9 +-
.../client/grpc/GrpcDirectGetWorkStreamTest.java | 27 +-
.../windmill/state/WindmillStateInternalsTest.java | 16 +-
.../fnexecution/environment/DockerCommand.java | 2 +-
.../PortablePipelineJarCreatorTest.java | 1 +
.../java/org/apache/beam/runners/jet/Utils.java | 6 +-
.../org/apache/beam/runners/jet/TestStreamP.java | 2 +-
.../beam/runners/samza/SamzaPipelineResult.java | 1 +
.../beam/runners/samza/runtime/OpMessage.java | 2 +-
.../samza/translation/TranslationContext.java | 4 +-
.../samza/adapter/BoundedSourceSystemTest.java | 3 +-
.../translation/EvaluationContext.java | 1 +
.../GroupNonMergingWindowsFunctions.java | 11 +-
.../spark/translation/TransformTranslator.java | 2 +-
.../streaming/StreamingTransformTranslator.java | 2 +-
.../runners/spark/util/SideInputBroadcast.java | 1 +
.../streaming/utils/EmbeddedKafkaCluster.java | 5 +-
sdks/go.mod | 36 +-
sdks/go.sum | 72 +-
sdks/go/pkg/beam/core/core.go | 2 +-
sdks/go/pkg/beam/core/metrics/sampler.go | 4 +-
sdks/go/pkg/beam/core/metrics/sampler_test.go | 4 +-
sdks/go/pkg/beam/options/jobopts/options.go | 4 +-
.../prism/internal/engine/elementmanager.go | 23 +-
.../beam/runners/prism/internal/environments.go | 8 +-
sdks/go/pkg/beam/runners/prism/internal/execute.go | 6 +-
sdks/go/pkg/beam/runners/prism/internal/stage.go | 13 +
.../apache/beam/sdk/jmh/schemas/RowBundles.java | 1 +
.../sdk/fn/data/BeamFnDataGrpcMultiplexer.java | 1 +
.../sdk/fn/data/BeamFnDataInboundObserver.java | 1 +
.../server/GrpcContextHeaderAccessorProvider.java | 2 +-
.../src/main/java/org/apache/beam/sdk/io/Read.java | 1 +
.../java/org/apache/beam/sdk/io/WriteFiles.java | 2 +-
.../beam/sdk/options/PipelineOptionsFactory.java | 7 +-
.../apache/beam/sdk/options/SdkHarnessOptions.java | 5 +-
.../beam/sdk/schemas/logicaltypes/OneOfType.java | 4 +-
.../apache/beam/sdk/schemas/transforms/Select.java | 1 +
.../org/apache/beam/sdk/testing/TestStream.java | 2 +-
.../beam/sdk/transforms/DoFnOutputReceivers.java | 4 +-
.../beam/sdk/transforms/DoFnSchemaInformation.java | 2 +-
.../apache/beam/sdk/transforms/MapElements.java | 1 +
.../java/org/apache/beam/sdk/transforms/Reify.java | 2 +-
.../java/org/apache/beam/sdk/transforms/Wait.java | 2 +-
.../beam/sdk/transforms/join/CoGbkResult.java | 2 +-
.../sdk/transforms/reflect/DoFnSignatures.java | 2 +-
.../splittabledofn/GrowableOffsetRangeTracker.java | 28 +-
.../org/apache/beam/sdk/util/CombineFnUtil.java | 2 +-
.../construction/GroupIntoBatchesTranslation.java | 2 +-
.../sdk/util/construction/ParDoTranslation.java | 2 +-
.../sdk/util/construction/SplittableParDo.java | 2 +-
.../sdk/util/construction/TransformUpgrader.java | 18 +-
.../sdk/util/construction/UnconsumedReads.java | 2 +-
.../main/java/org/apache/beam/sdk/values/Row.java | 3 +-
.../org/apache/beam/sdk/coders/ZstdCoderTest.java | 1 +
.../org/apache/beam/sdk/io/CountingSourceTest.java | 2 +-
.../org/apache/beam/sdk/io/FileSystemsTest.java | 1 +
.../sdk/options/PipelineOptionsFactoryTest.java | 2 +-
.../beam/sdk/schemas/AutoValueSchemaTest.java | 2 +-
.../beam/sdk/schemas/JavaBeanSchemaTest.java | 2 +-
.../beam/sdk/schemas/JavaFieldSchemaTest.java | 4 +-
.../beam/sdk/schemas/transforms/GroupTest.java | 4 +-
.../beam/sdk/schemas/utils/TestJavaBeans.java | 4 +-
.../beam/sdk/transforms/GroupIntoBatchesTest.java | 4 +-
.../util/construction/CombineTranslationTest.java | 4 +-
.../beam/sdk/values/EncodableThrowableTest.java | 5 +-
.../beam/sdk/values/TypeDescriptorsTest.java | 1 +
sdks/java/expansion-service/container/Dockerfile | 2 +
sdks/java/expansion-service/container/build.gradle | 4 +
.../sdk/expansion/service/ExpansionService.java | 2 +-
.../extensions/protobuf/ProtoByteBuddyUtils.java | 4 +-
.../sketching/ApproximateDistinctTest.java | 2 +-
.../extensions/sketching/TDigestQuantilesTest.java | 2 +-
sdks/java/extensions/sql/build.gradle | 8 +-
sdks/java/extensions/sql/hcatalog/build.gradle | 2 +-
sdks/java/extensions/sql/iceberg/build.gradle | 81 ++
.../sql/meta/provider/iceberg/IcebergCatalog.java | 0
.../provider/iceberg/IcebergCatalogRegistrar.java} | 12 +-
.../sql/meta/provider/iceberg/IcebergFilter.java | 0
.../sql/meta/provider/iceberg/IcebergTable.java | 0
.../provider/iceberg/IcebergTableProvider.java | 0
.../sql/meta/provider/iceberg/package-info.java | 0
.../provider/iceberg/BeamSqlCliIcebergTest.java | 0
.../meta/provider/iceberg/IcebergFilterTest.java | 0
.../meta/provider/iceberg/IcebergReadWriteIT.java | 6 +-
.../provider/iceberg/IcebergTableProviderTest.java | 0
.../meta/provider/iceberg}/PubsubToIcebergIT.java | 3 +-
.../sql/impl/transform/agg/CovarianceFn.java | 4 +-
.../sql/impl/transform/agg/VarianceFn.java | 4 +-
.../sql/meta/catalog/InMemoryCatalogRegistrar.java | 6 +-
.../sql/meta/provider/mongodb/MongoDbTable.java | 19 +-
...ittableTruncateSizedRestrictionsDoFnRunner.java | 2 +-
.../fn/harness/control/ExecutionStateSampler.java | 13 +-
.../fn/harness/control/ProcessBundleHandler.java | 4 +-
.../harness/logging/BeamFnLoggingClientTest.java | 5 +-
.../apache/beam/sdk/io/aws2/kinesis/KinesisIO.java | 2 +-
.../apache/beam/sdk/io/aws2/StaticSupplier.java | 1 +
.../io/cdap/context/FailureCollectorWrapper.java | 2 +-
.../cdap/context/FailureCollectorWrapperTest.java | 9 +-
.../beam/sdk/io/common/DatabaseTestHelper.java | 2 +-
.../org/apache/beam/sdk/io/common/IOITHelper.java | 2 +-
.../DebeziumReadSchemaTransformProvider.java | 3 +-
sdks/java/io/expansion-service/build.gradle | 20 +-
.../FileWriteSchemaTransformFormatProviders.java | 2 +-
.../sdk/io/fileschematransform/XmlRowAdapter.java | 1 +
.../beam/sdk/io/googleads/GoogleAdsIOTest.java | 1 +
.../beam/sdk/io/gcp/bigquery/BigQueryHelpers.java | 17 +
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 3 +-
.../sdk/io/gcp/bigquery/BigQuerySourceBase.java | 3 +-
.../beam/sdk/io/gcp/datastore/DatastoreV1.java | 6 +-
.../apache/beam/sdk/io/gcp/healthcare/FhirIO.java | 1 -
.../apache/beam/sdk/io/gcp/pubsub/PubsubIO.java | 5 +-
.../beam/sdk/io/gcp/pubsub/PubsubTestClient.java | 4 +-
.../gcp/pubsublite/internal/CloserReference.java | 2 +-
.../gcp/pubsublite/internal/MemoryLimiterImpl.java | 1 +
.../beam/sdk/io/gcp/spanner/SpannerAccessor.java | 23 +-
.../beam/sdk/io/gcp/spanner/SpannerConfig.java | 44 +
.../apache/beam/sdk/io/gcp/spanner/SpannerIO.java | 169 ++-
.../io/gcp/spanner/SpannerTransformRegistrar.java | 22 +
.../MetadataSpannerConfigFactory.java | 10 +
.../action/DetectNewPartitionsAction.java | 2 +-
.../action/QueryChangeStreamAction.java | 2 +-
.../sdk/io/gcp/testing/FakeDatasetService.java | 3 +-
.../BigtableReadSchemaTransformProviderIT.java | 2 +-
.../beam/sdk/io/gcp/datastore/V1TestUtil.java | 5 +-
.../beam/sdk/io/gcp/pubsub/PubsubIOTest.java | 38 +-
.../beam/sdk/io/gcp/spanner/SpannerIOReadTest.java | 2 +-
.../beam/sdk/io/gcp/spanner/SpannerReadIT.java | 55 -
.../changestreams/it/IntegrationTestEnv.java | 1 -
.../beam/sdk/io/hbase/HBaseRowMutationsCoder.java | 4 +-
sdks/java/io/iceberg/build.gradle | 12 +-
.../sdk/io/iceberg/RecordWriterManagerTest.java | 40 +-
sdks/java/io/jdbc/build.gradle | 3 +
.../java/org/apache/beam/sdk/io/jdbc/JdbcIO.java | 2 +
.../io/jdbc/JdbcReadSchemaTransformProvider.java | 18 +
.../io/jdbc/JdbcWriteSchemaTransformProvider.java | 18 +
.../PostgresSchemaTransformTranslation.java} | 28 +-
.../ReadFromPostgresSchemaTransformProvider.java | 48 +-
.../WriteToPostgresSchemaTransformProvider.java | 38 +-
.../apache/beam/sdk/io/jdbc/JdbcIOPostgresIT.java | 178 +++
.../PostgresSchemaTransformTranslationTest.java} | 123 +-
.../java/org/apache/beam/sdk/io/jms/JmsIO.java | 1 +
sdks/java/io/kafka/build.gradle | 3 +
.../sdk/io/kafka/jmh/KafkaIOUtilsBenchmark.java | 1 +
sdks/java/io/kafka/kafka-integration-test.gradle | 1 +
.../beam/sdk/io/kafka/KafkaCommitOffset.java | 2 +
.../org/apache/beam/sdk/io/kafka/KafkaIOUtils.java | 19 +-
.../kafka/KafkaWriteSchemaTransformProvider.java | 132 +-
.../org/apache/beam/sdk/io/kafka/KafkaWriter.java | 2 +-
.../beam/sdk/io/kafka/ReadFromKafkaDoFn.java | 194 ++-
.../org/apache/beam/sdk/io/kafka/KafkaIOIT.java | 5 +-
.../org/apache/beam/sdk/io/kafka/KafkaIOTest.java | 23 +-
.../KafkaWriteSchemaTransformProviderTest.java | 44 +-
.../java/org/apache/beam/sdk/io/mqtt/MqttIO.java | 84 +-
.../org/apache/beam/sdk/io/mqtt/MqttIOTest.java | 59 +-
.../apache/beam/sdk/io/pulsar/PulsarIOUtils.java | 1 -
.../beam/sdk/io/pulsar/ReadFromPulsarDoFn.java | 1 -
.../apache/beam/sdk/io/pulsar/PulsarIOTest.java | 2 +
.../sparkreceiver/RabbitMqReceiverWithOffset.java | 2 +-
.../sdk/io/sparkreceiver/SparkReceiverIOIT.java | 2 +-
.../beam/sdk/io/splunk/SplunkEventWriter.java | 6 +-
.../beam/sdk/io/synthetic/BundleSplitterTest.java | 4 +-
.../java/org/apache/beam/sdk/managed/Managed.java | 3 +
.../testutils/publishing/InfluxDBPublisher.java | 2 +-
sdks/python/apache_beam/coders/coder_impl.pxd | 4 +
sdks/python/apache_beam/coders/coder_impl.py | 66 +-
sdks/python/apache_beam/coders/coders.py | 72 +-
.../apache_beam/coders/coders_test_common.py | 146 ++-
sdks/python/apache_beam/coders/typecoders.py | 1 +
.../snippets/transforms/elementwise/enrichment.py | 211 ++++
.../transforms/elementwise/enrichment_test.py | 263 +++-
.../apache_beam/internal/cloudpickle_pickler.py | 2 +
.../io/components/adaptive_throttler.py | 92 ++
sdks/python/apache_beam/io/gcp/pubsub.py | 83 +-
.../apache_beam/io/gcp/pubsub_integration_test.py | 85 ++
sdks/python/apache_beam/io/gcp/pubsub_test.py | 65 +-
sdks/python/apache_beam/io/jdbc.py | 92 +-
sdks/python/apache_beam/ml/inference/base.py | 43 +-
sdks/python/apache_beam/ml/transforms/base.py | 39 +
sdks/python/apache_beam/ml/transforms/base_test.py | 117 ++
.../ml/transforms/embeddings/vertex_ai.py | 241 +++-
.../ml/transforms/embeddings/vertex_ai_test.py | 107 ++
.../python/apache_beam/options/pipeline_options.py | 45 +-
sdks/python/apache_beam/pipeline.py | 9 +
.../runners/dataflow/dataflow_runner.py | 8 +
.../runners/dataflow/ptransform_overrides.py | 65 +-
.../apache_beam/runners/direct/direct_runner.py | 84 +-
.../yaml_parse_utils.py | 176 +++
.../apache-beam-jupyterlab-sidepanel/package.json | 17 +-
.../src/SidePanel.ts | 11 +-
.../apache-beam-jupyterlab-sidepanel/src/index.ts | 35 +-
.../src/yaml/CustomStyle.tsx | 179 +++
.../src/yaml/DataType.ts | 32 +-
.../src/yaml/EditablePanel.tsx | 408 ++++++
.../src/yaml/EmojiMap.ts | 75 ++
.../src/yaml/Yaml.tsx | 322 +++++
.../src/yaml/YamlEditor.tsx | 338 +++++
.../src/yaml/YamlFlow.tsx | 227 ++++
.../ClustersWidget.tsx => yaml/YamlWidget.tsx} | 6 +-
.../style/index.css | 3 +
.../style/mdc-theme.css | 4 +-
.../style/{index.css => yaml/Yaml.css} | 30 +-
.../style/{index.css => yaml/YamlEditor.css} | 26 +-
.../style/yaml/YamlFlow.css | 168 +++
.../apache-beam-jupyterlab-sidepanel/yarn.lock | 535 +++++++-
.../apache_beam/runners/portability/job_server.py | 7 +-
.../runners/portability/prism_runner.py | 54 +
.../runners/worker/data_sampler_test.py | 42 +-
.../apache_beam/runners/worker/worker_status.py | 10 +-
.../apache_beam/tools/coders_microbenchmark.py | 10 +
sdks/python/apache_beam/transforms/core.py | 84 +-
sdks/python/apache_beam/transforms/core_test.py | 134 ++
.../transforms/enrichment_handlers/cloudsql.py | 30 +-
.../enrichment_handlers/cloudsql_it_test.py | 35 +-
sdks/python/apache_beam/transforms/external.py | 7 +-
sdks/python/apache_beam/transforms/managed.py | 7 +-
.../apache_beam/transforms/ptransform_test.py | 86 +-
sdks/python/apache_beam/transforms/trigger.py | 119 +-
sdks/python/apache_beam/transforms/trigger_test.py | 50 -
sdks/python/apache_beam/transforms/util.py | 20 +-
sdks/python/apache_beam/transforms/util_test.py | 62 +
sdks/python/apache_beam/typehints/schemas.py | 92 ++
sdks/python/apache_beam/utils/subprocess_server.py | 23 +-
sdks/python/apache_beam/version.py | 2 +-
sdks/python/apache_beam/yaml/README.md | 4 +-
sdks/python/apache_beam/yaml/examples/README.md | 10 +-
.../yaml/examples/testing/examples_test.py | 6 +-
.../iceberg_to_alloydb.yaml} | 27 +-
.../transforms/blueprint/pubsub_to_iceberg.yaml | 6 +-
.../examples/transforms/jinja/import/README.md | 8 +-
.../examples/transforms/jinja/include/README.md | 8 +-
.../transforms/ml/fraud_detection}/README.md | 10 +-
.../fraud_detection_mlops_beam_yaml_sdk.ipynb | 1329 ++++++++++++++++++++
.../transforms/ml/sentiment_analysis/README.md | 21 +-
.../streaming_sentiment_analysis.yaml | 29 +-
.../examples/transforms/ml/taxi_fare/README.md | 21 +-
.../custom_nyc_taxifare_model_deployment.ipynb | 2 +-
.../taxi_fare/streaming_taxifare_prediction.yaml | 2 +-
.../databases/iceberg.yaml} | 28 +-
sdks/python/apache_beam/yaml/json_utils.py | 7 +-
sdks/python/apache_beam/yaml/pipeline.schema.yaml | 3 +
.../apache_beam/yaml/tests/assign_timestamps.yaml | 87 ++
sdks/python/apache_beam/yaml/tests/create.yaml | 23 +
.../yaml/tests/validate_with_schema.yaml | 46 +-
sdks/python/apache_beam/yaml/yaml_errors.py | 5 +-
sdks/python/apache_beam/yaml/yaml_mapping.py | 4 +-
sdks/python/apache_beam/yaml/yaml_provider.py | 9 +-
sdks/python/apache_beam/yaml/yaml_transform.py | 271 ++++
.../python/apache_beam/yaml/yaml_transform_test.py | 114 ++
.../apache_beam/yaml/yaml_transform_unit_test.py | 111 ++
sdks/python/container/common.gradle | 4 +-
sdks/python/container/run_generate_requirements.sh | 2 +-
sdks/python/setup.py | 4 +-
sdks/python/test-suites/direct/common.gradle | 28 -
sdks/python/test-suites/portable/common.gradle | 2 +-
sdks/python/tox.ini | 2 +-
sdks/typescript/package.json | 2 +-
settings.gradle.kts | 4 +
.../content/en/documentation/programming-guide.md | 98 ++
.../content/en/documentation/sdks/yaml-errors.md | 10 +-
.../content/en/documentation/sdks/yaml-schema.md | 131 ++
.../python/elementwise/enrichment-cloudsql.md | 146 +++
.../transforms/python/elementwise/enrichment.md | 1 +
website/www/site/data/capability_matrix.yaml | 158 +++
.../partials/section-menu/en/documentation.html | 1 +
363 files changed, 9964 insertions(+), 1985 deletions(-)
copy .github/workflows/{beam_Infrastructure_UsersPermissions.yml =>
beam_Infrastructure_ServiceAccountKeys.yml} (54%)
copy examples/java/{sql => iceberg}/build.gradle (54%)
rename examples/java/{src/main/java/org/apache/beam/examples/cookbook =>
iceberg/src/main/java/org/apache/beam/examples/iceberg}/IcebergBatchWriteExample.java
(99%)
rename examples/java/{src/main/java/org/apache/beam/examples/cookbook =>
iceberg/src/main/java/org/apache/beam/examples/iceberg}/IcebergRestCatalogCDCExample.java
(99%)
rename examples/java/{src/main/java/org/apache/beam/examples/cookbook =>
iceberg/src/main/java/org/apache/beam/examples/iceberg}/IcebergRestCatalogStreamingWriteExample.java
(99%)
rename examples/java/{src/main/java/org/apache/beam/examples/cookbook =>
iceberg/src/main/java/org/apache/beam/examples/iceberg}/IcebergTaxiExamples.java
(99%)
create mode 100644 examples/yaml/README.md
delete mode 100644 infra/keys/gcp_logger.py
create mode 100644 sdks/java/extensions/sql/iceberg/build.gradle
rename sdks/java/extensions/sql/{ =>
iceberg}/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/iceberg/IcebergCatalog.java
(100%)
copy
sdks/java/extensions/sql/{src/main/java/org/apache/beam/sdk/extensions/sql/meta/catalog/InMemoryCatalogRegistrar.java
=>
iceberg/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/iceberg/IcebergCatalogRegistrar.java}
(74%)
rename sdks/java/extensions/sql/{ =>
iceberg}/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/iceberg/IcebergFilter.java
(100%)
rename sdks/java/extensions/sql/{ =>
iceberg}/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/iceberg/IcebergTable.java
(100%)
rename sdks/java/extensions/sql/{ =>
iceberg}/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/iceberg/IcebergTableProvider.java
(100%)
rename sdks/java/extensions/sql/{ =>
iceberg}/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/iceberg/package-info.java
(100%)
rename sdks/java/extensions/sql/{ =>
iceberg}/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/iceberg/BeamSqlCliIcebergTest.java
(100%)
rename sdks/java/extensions/sql/{ =>
iceberg}/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/iceberg/IcebergFilterTest.java
(100%)
rename sdks/java/extensions/sql/{ =>
iceberg}/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/iceberg/IcebergReadWriteIT.java
(98%)
rename sdks/java/extensions/sql/{ =>
iceberg}/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/iceberg/IcebergTableProviderTest.java
(100%)
rename
sdks/java/extensions/sql/{src/test/java/org/apache/beam/sdk/extensions/sql =>
iceberg/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/iceberg}/PubsubToIcebergIT.java
(98%)
copy
sdks/java/io/{kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaSchemaTransformTranslation.java
=>
jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/providers/PostgresSchemaTransformTranslation.java}
(72%)
create mode 100644
sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcIOPostgresIT.java
copy
sdks/java/io/{kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaSchemaTransformTranslationTest.java
=>
jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/providers/PostgresSchemaTransformTranslationTest.java}
(60%)
create mode 100644
sdks/python/apache_beam/runners/interactive/extensions/apache-beam-jupyterlab-sidepanel/apache_beam_jupyterlab_sidepanel/yaml_parse_utils.py
create mode 100644
sdks/python/apache_beam/runners/interactive/extensions/apache-beam-jupyterlab-sidepanel/src/yaml/CustomStyle.tsx
copy website/www/site/assets/js/shuffle-elements.js =>
sdks/python/apache_beam/runners/interactive/extensions/apache-beam-jupyterlab-sidepanel/src/yaml/DataType.ts
(57%)
create mode 100644
sdks/python/apache_beam/runners/interactive/extensions/apache-beam-jupyterlab-sidepanel/src/yaml/EditablePanel.tsx
create mode 100644
sdks/python/apache_beam/runners/interactive/extensions/apache-beam-jupyterlab-sidepanel/src/yaml/EmojiMap.ts
create mode 100644
sdks/python/apache_beam/runners/interactive/extensions/apache-beam-jupyterlab-sidepanel/src/yaml/Yaml.tsx
create mode 100644
sdks/python/apache_beam/runners/interactive/extensions/apache-beam-jupyterlab-sidepanel/src/yaml/YamlEditor.tsx
create mode 100644
sdks/python/apache_beam/runners/interactive/extensions/apache-beam-jupyterlab-sidepanel/src/yaml/YamlFlow.tsx
copy
sdks/python/apache_beam/runners/interactive/extensions/apache-beam-jupyterlab-sidepanel/src/{clusters/ClustersWidget.tsx
=> yaml/YamlWidget.tsx} (86%)
copy
sdks/python/apache_beam/runners/interactive/extensions/apache-beam-jupyterlab-sidepanel/style/{index.css
=> yaml/Yaml.css} (58%)
copy
sdks/python/apache_beam/runners/interactive/extensions/apache-beam-jupyterlab-sidepanel/style/{index.css
=> yaml/YamlEditor.css} (61%)
create mode 100644
sdks/python/apache_beam/runners/interactive/extensions/apache-beam-jupyterlab-sidepanel/style/yaml/YamlFlow.css
copy sdks/python/apache_beam/yaml/examples/transforms/{io/iceberg_read.yaml =>
blueprint/iceberg_to_alloydb.yaml} (68%)
copy sdks/python/{test-suites/containers/tensorrt_runinference =>
apache_beam/yaml/examples/transforms/ml/fraud_detection}/README.md (68%)
create mode 100644
sdks/python/apache_beam/yaml/examples/transforms/ml/fraud_detection/fraud_detection_mlops_beam_yaml_sdk.ipynb
copy sdks/python/apache_beam/yaml/{tests/avro.yaml =>
extended_tests/databases/iceberg.yaml} (68%)
create mode 100644
website/www/site/content/en/documentation/sdks/yaml-schema.md
create mode 100644
website/www/site/content/en/documentation/transforms/python/elementwise/enrichment-cloudsql.md
