This is an automated email from the ASF dual-hosted git repository.
Amar3tto pushed a change to branch isolate-py314-sdk-snapshots
in repository https://gitbox.apache.org/repos/asf/beam.git
from f4b13a08539 Bump pillow (#39396)
add f80294498cf Bump actions/checkout from 4 to 7 (#39399)
add 7735b396afb Bump pillow (#39395)
add ab36569659a fix GCP_PATH expansion (#39400)
add dca1253c7e3 Fix CdapIO dependency (#39393)
add 7bc2fb49f8d Fix a memory leak in the failed instruction id cache.
(#39405)
add 5b441f27082 Update CHANGES.md to document a memory growth issue.
(#39407)
add 202d339ad9f Fix OpenSearch env fallback for username/password in
examples
add c54595b4b9f Merge pull request #39111: Fix OpenSearch
username/password env fallback in examples
add 22b2f385a8a Refactor LoadTestBase
add 875cfd11ced Refactor DefaultPipelineLauncher
add 5312b03502c Add IcebergIOLT
add 7fb07784bf2 Add looker metrics for IcebergIO
add 9432acebb2b Refactoring
add ab142e93fb6 Fix conflict
add eccd9b78d50 Merge pull request #39270 from apache/icerbergio-benchmark
add 9baa19ad5ca Increase CI timeouts for Python 3.14 multiarch container
jobs (#39401)
add 0c151d8ddb2 Fix gRPC stream observer leak on ProcessBundleHandler
shutdown (#39390)
add 414b48ad644 add Apache third party allow list check and doc (#39392)
add 83fcb2f9cdf [Gemini] Add Java GeminiModelHandler class (#39245)
add 55e1ecbb138 Add metric for Kafka offset commit failures (#38889)
add 51b2ad6bc08 Add drain states to PipelineResult (#39020)
add ee44102d01b Bump github.com/aws/aws-sdk-go-v2/credentials in /sdks
(#39425)
add 0dd47882822 Interview with Raj Katakam from Intuit Credit Karma
(#39412)
add 22a59516168 Document job management and interactive API protos (#39294)
add d2d7ef88eac Prefer binary wheels when installing Python container
dependencies (#39402)
add 3f3f18a8ce0 Bump github.com/aws/aws-sdk-go-v2/feature/s3/manager in
/sdks (#39421)
add aa8869c633d Bump cloud.google.com/go/storage from 1.63.1 to 1.64.0 in
/sdks (#39422)
add 583601b13bf Fix pyrefly check bad-context-manager (#39418)
add aaad9b4b8e6 [Experimental] Use zstd compression in Docker builds
(#39409)
add abe3fc77c3d Fix silent per-char iteration when DoFn returns
str/bytes/dict (#38429)
add a15b880c53c Merge pull request #39043: Improve WithKeys coder
inference context
add b04ca1e0cee Fix pyrefly check bad-specialization (#39419)
add c8a689f7c0d Fix pyrefly check bad-argument-count (#39416)
add da7693c3dd1 [Gemini] Fix pyrefly check invalid-yield (#39417)
add f1d9c3e122e docs: Document pipeline runner API protos (#39292)
add adbb11ddb46 Fix gcs endpoint pipeline option wiring (#39435)
add 5e499388670 KeyCommitTooLarge logging improvements (#39316)
No new revisions were added by this update.
Summary of changes:
.github/actions/setup-k8s-access/action.yml | 2 +-
...tCommit_Python_ValidatesContainer_Dataflow.json | 2 +-
.../beam_PreCommit_Flink_Container.json | 2 +-
.github/workflows/README.md | 9 +
.../beam_Infrastructure_UsersPermissions.yml | 2 +-
.../beam_PostCommit_Java_IO_Performance_Tests.yml | 2 +-
.github/workflows/beam_PostCommit_Python_Arm.yml | 3 +-
.github/workflows/beam_PreCommit_GHA.yml | 12 +-
...beam_Python_ValidatesContainer_Dataflow_ARM.yml | 2 +
.github/workflows/build_wheels.yml | 3 +-
.github/workflows/codeql.yml | 2 +-
.github/workflows/update_python_dependencies.yml | 1 +
.test-infra/tools/refresh_looker_metrics.py | 2 +
CHANGES.md | 29 +-
.../org/apache/beam/gradle/BeamDockerPlugin.groovy | 24 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 2 +-
.../beam-ml/rag_usecase/opensearch_connector.py | 8 +-
.../beam-ml/rag_usecase/opensearch_enrichment.py | 4 +-
it/build.gradle | 6 +-
it/clickhouse/build.gradle | 8 +-
it/common/build.gradle | 29 +-
.../apache/beam/it/common}/artifacts/Artifact.java | 2 +-
.../beam/it/common}/artifacts/ArtifactClient.java | 2 +-
.../beam/it/common}/artifacts/GcsArtifact.java | 2 +-
.../beam/it/common}/artifacts/package-info.java | 2 +-
.../it/common}/artifacts/utils/ArtifactUtils.java | 2 +-
.../it/common}/artifacts/utils/AvroTestUtil.java | 2 +-
.../it/common}/artifacts/utils/JsonTestUtil.java | 2 +-
.../common}/artifacts/utils/ParquetTestUtil.java | 2 +-
.../it/common}/artifacts/utils/package-info.java | 2 +-
.../common}/bigquery/BigQueryResourceManager.java | 2 +-
.../bigquery/BigQueryResourceManagerException.java | 2 +-
.../bigquery/BigQueryResourceManagerUtils.java | 2 +-
.../beam/it/common/bigquery}/package-info.java | 4 +-
.../common}/dataflow/AbstractPipelineLauncher.java | 2 +-
.../common}/dataflow/DefaultPipelineLauncher.java | 52 +--
.../beam/it/common}/dataflow/package-info.java | 2 +-
.../it/common}/monitoring/MonitoringClient.java | 2 +-
.../beam/it/common}/monitoring/package-info.java | 2 +-
.../it/common}/storage/GcsResourceManager.java | 10 +-
.../beam/it/common}/storage/package-info.java | 2 +-
.../src/main/resources/test-artifact.json | 0
.../beam/it/common}/artifacts/GcsArtifactTest.java | 4 +-
.../common}/artifacts/utils/ArtifactUtilsTest.java | 4 +-
.../bigquery/BigQueryResourceManagerTest.java | 2 +-
.../bigquery/BigQueryResourceManagerUtilsTest.java | 4 +-
.../dataflow/AbstractPipelineLauncherTest.java | 2 +-
.../dataflow/DefaultPipelineLauncherTest.java | 8 +-
.../beam/it/common/dataflow}/IOLoadTestBase.java | 21 +-
.../beam/it/common/dataflow}/IOStressTestBase.java | 2 +-
.../beam/it/common/dataflow}/LoadTestBase.java | 8 +-
.../it/common}/storage/GcsResourceManagerTest.java | 19 +-
it/google-cloud-platform/build.gradle | 21 +-
.../it/gcp/artifacts/matchers/package-info.java | 20 --
.../gcp/bigquery/conditions/BigQueryRowsCheck.java | 2 +-
.../it/gcp/dataflow/ClassicTemplateClient.java | 1 +
.../beam/it/gcp/dataflow/FlexTemplateClient.java | 1 +
.../beam/it/gcp/datagenerator/package-info.java | 20 --
.../it/gcp/spanner/matchers/SpannerAsserts.java | 2 +-
.../java/org/apache/beam/it/gcp/WordCountIT.java | 1 +
.../apache/beam/it/gcp/bigquery/BigQueryIOLT.java | 4 +-
.../apache/beam/it/gcp/bigquery/BigQueryIOST.java | 4 +-
.../beam/it/gcp/bigquery/BigQueryStreamingLT.java | 2 +-
.../apache/beam/it/gcp/bigtable/BigTableIOLT.java | 3 +-
.../apache/beam/it/gcp/bigtable/BigTableIOST.java | 3 +-
.../it/gcp/dataflow/ClassicTemplateClientTest.java | 9 +-
.../it/gcp/dataflow/FlexTemplateClientTest.java | 8 +-
.../beam/it/gcp/datagenerator/DataGenerator.java | 2 +-
.../org/apache/beam/it/gcp/pubsub/PubSubIOST.java | 3 +-
.../org/apache/beam/it/gcp/pubsub/PubsubIOLT.java | 3 +-
.../apache/beam/it/gcp/spanner/SpannerIOLT.java | 3 +-
.../apache/beam/it/gcp/spanner/SpannerIOST.java | 3 +-
.../apache/beam/it/gcp/storage/FileBasedIOLT.java | 4 +-
it/iceberg/build.gradle | 134 ++++++++
.../org/apache/beam/it/iceberg/IcebergIOLT.java | 363 +++++++++++++++++++++
it/kafka/build.gradle | 3 +-
.../java/org/apache/beam/it/kafka/KafkaIOLT.java | 3 +-
.../java/org/apache/beam/it/kafka/KafkaIOST.java | 3 +-
it/truthmatchers/build.gradle | 6 +-
.../beam/it/truthmatchers}/ArtifactAsserts.java | 15 +-
.../beam/it/truthmatchers}/ArtifactsSubject.java | 21 +-
.../interactive/v1/beam_interactive_api.proto | 4 +
.../job_management/v1/beam_artifact_api.proto | 19 +-
.../job_management/v1/beam_expansion_api.proto | 9 +
.../model/job_management/v1/beam_job_api.proto | 32 ++
.../beam/model/pipeline/v1/beam_runner_api.proto | 94 +++++-
.../apache/beam/model/pipeline/v1/endpoints.proto | 3 +
.../model/pipeline/v1/external_transforms.proto | 20 ++
.../apache/beam/model/pipeline/v1/metrics.proto | 73 ++++-
.../org/apache/beam/model/pipeline/v1/schema.proto | 94 ++++++
.../runners/flink/FlinkDetachedRunnerResult.java | 4 +-
.../beam/runners/flink/FlinkRunnerResultTest.java | 12 +-
.../beam/runners/dataflow/DataflowPipelineJob.java | 1 +
.../beam/runners/dataflow/DataflowRunner.java | 3 +-
.../beam/runners/dataflow/util/MonitoringUtil.java | 6 +-
.../runners/dataflow/DataflowPipelineJobTest.java | 2 +-
.../beam/runners/dataflow/DataflowRunnerTest.java | 26 +-
.../runners/dataflow/util/MonitoringUtilTest.java | 4 +-
.../worker/StreamingModeExecutionContext.java | 53 +++
.../streaming/KeyCommitTooLargeException.java | 20 +-
.../processing/ComputationWorkExecutorFactory.java | 14 +-
.../work/processing/StreamingWorkScheduler.java | 61 +---
.../worker/StreamingDataflowWorkerTest.java | 31 +-
.../worker/StreamingModeExecutionContextTest.java | 5 +
.../dataflow/worker/WorkerCustomSourcesTest.java | 8 +
.../beam/runners/jobsubmission/JobInvocation.java | 15 +-
.../runners/jobsubmission/JobInvocationTest.java | 36 ++
.../portability/JobServicePipelineResult.java | 7 +-
.../runners/portability/PortableRunnerTest.java | 21 ++
sdks/go.mod | 34 +-
sdks/go.sum | 68 ++--
.../java/org/apache/beam/sdk/PipelineResult.java | 6 +
.../org/apache/beam/sdk/transforms/WithKeys.java | 40 ++-
.../apache/beam/sdk/transforms/WithKeysTest.java | 15 +
.../beam/sdk/extensions/gcp/util/GcsUtilV1.java | 17 +-
.../beam/sdk/extensions/gcp/util/GcsUtilTest.java | 13 +
.../fn/harness/control/ProcessBundleHandler.java | 1 +
.../beam/fn/harness/data/BeamFnDataClient.java | 9 +-
.../beam/fn/harness/data/BeamFnDataGrpcClient.java | 12 +
sdks/java/io/cdap/build.gradle | 5 +
.../beam/sdk/io/kafka/KafkaCommitOffset.java | 6 +
.../ml/inference/{openai => gemini}/build.gradle | 19 +-
.../ml/inference/gemini/GeminiImageResponse.java | 17 +-
.../inference/gemini/GeminiInferenceFunctions.java | 67 ++++
.../ml/inference/gemini/GeminiModelHandler.java | 86 +++++
.../ml/inference/gemini/GeminiModelParameters.java | 60 ++++
.../ml/inference/gemini/GeminiRequestFunction.java | 27 +-
.../sdk/ml/inference/gemini/GeminiStringInput.java | 17 +-
.../ml/inference/gemini/GeminiStringResponse.java | 17 +-
.../sdk/ml/inference/gemini}/package-info.java | 4 +-
.../ml/inference/gemini/GeminiModelHandlerIT.java | 163 +++++++++
.../inference/gemini/GeminiModelHandlerTest.java | 133 ++++++++
.../ml/inference/remote/PredictionResultCoder.java | 74 +++++
.../sdk/ml/inference/remote/RemoteInference.java | 34 +-
.../ml/inference/remote/RemoteInferenceTest.java | 4 +-
.../apache/beam/sdk/nexmark/NexmarkLauncher.java | 2 +
...ytorch_image_classification_with_side_inputs.py | 2 +-
.../kfp/components/train/requirements.txt | 2 +-
.../apache_beam/io/gcp/bigquery_change_history.py | 10 +-
sdks/python/apache_beam/ml/anomaly/specifiable.py | 2 +-
sdks/python/apache_beam/ml/inference/base.py | 3 +-
sdks/python/apache_beam/runners/common.pxd | 1 +
sdks/python/apache_beam/runners/common.py | 35 ++
sdks/python/apache_beam/runners/common_test.py | 58 ++++
.../runners/interactive/recording_manager.py | 2 +-
.../runners/portability/prism_runner.py | 5 +-
.../apache_beam/runners/worker/sdk_worker.py | 12 +-
.../apache_beam/runners/worker/sdk_worker_test.py | 38 +++
sdks/python/container/Dockerfile | 3 +-
sdks/python/container/run_generate_requirements.sh | 4 +-
sdks/python/pyproject.toml | 4 -
settings.gradle.kts | 2 +
.../beam-summit-2026-interview-with-raj-katakam.md | 88 +++++
website/www/site/content/en/performance/_index.md | 1 +
.../content/en/performance/icebergio/_index.md | 50 +++
website/www/site/data/authors.yml | 3 +
website/www/site/data/performance.yaml | 31 ++
157 files changed, 2501 insertions(+), 477 deletions(-)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/artifacts/Artifact.java (97%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/artifacts/ArtifactClient.java
(99%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/artifacts/GcsArtifact.java (96%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/artifacts/package-info.java
(95%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/artifacts/utils/ArtifactUtils.java
(98%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/artifacts/utils/AvroTestUtil.java
(98%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/artifacts/utils/JsonTestUtil.java
(99%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/artifacts/utils/ParquetTestUtil.java
(98%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/artifacts/utils/package-info.java
(94%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/bigquery/BigQueryResourceManager.java
(99%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/bigquery/BigQueryResourceManagerException.java
(96%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/bigquery/BigQueryResourceManagerUtils.java
(98%)
copy
{examples/java/cdap/hubspot/src/main/java/org/apache/beam/examples/complete/cdap/hubspot/options
=>
it/common/src/main/java/org/apache/beam/it/common/bigquery}/package-info.java
(86%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/dataflow/AbstractPipelineLauncher.java
(99%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/dataflow/DefaultPipelineLauncher.java
(92%)
copy it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/dataflow/package-info.java (95%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/monitoring/MonitoringClient.java
(99%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/monitoring/package-info.java
(94%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/storage/GcsResourceManager.java
(97%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/storage/package-info.java (95%)
rename it/{google-cloud-platform =>
common}/src/main/resources/test-artifact.json (100%)
rename it/{google-cloud-platform/src/test/java/org/apache/beam/it/gcp =>
common/src/test/java/org/apache/beam/it/common}/artifacts/GcsArtifactTest.java
(94%)
rename it/{google-cloud-platform/src/test/java/org/apache/beam/it/gcp =>
common/src/test/java/org/apache/beam/it/common}/artifacts/utils/ArtifactUtilsTest.java
(94%)
rename it/{google-cloud-platform/src/test/java/org/apache/beam/it/gcp =>
common/src/test/java/org/apache/beam/it/common}/bigquery/BigQueryResourceManagerTest.java
(99%)
rename it/{google-cloud-platform/src/test/java/org/apache/beam/it/gcp =>
common/src/test/java/org/apache/beam/it/common}/bigquery/BigQueryResourceManagerUtilsTest.java
(93%)
rename it/{google-cloud-platform/src/test/java/org/apache/beam/it/gcp =>
common/src/test/java/org/apache/beam/it/common}/dataflow/AbstractPipelineLauncherTest.java
(99%)
rename it/{google-cloud-platform/src/test/java/org/apache/beam/it/gcp =>
common/src/test/java/org/apache/beam/it/common}/dataflow/DefaultPipelineLauncherTest.java
(91%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/test/java/org/apache/beam/it/common/dataflow}/IOLoadTestBase.java
(87%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/test/java/org/apache/beam/it/common/dataflow}/IOStressTestBase.java
(99%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/test/java/org/apache/beam/it/common/dataflow}/LoadTestBase.java (98%)
rename it/{google-cloud-platform/src/test/java/org/apache/beam/it/gcp =>
common/src/test/java/org/apache/beam/it/common}/storage/GcsResourceManagerTest.java
(95%)
delete mode 100644
it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/matchers/package-info.java
delete mode 100644
it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datagenerator/package-info.java
rename it/google-cloud-platform/src/{main =>
test}/java/org/apache/beam/it/gcp/datagenerator/DataGenerator.java (99%)
create mode 100644 it/iceberg/build.gradle
create mode 100644
it/iceberg/src/test/java/org/apache/beam/it/iceberg/IcebergIOLT.java
rename
it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/matchers
=>
truthmatchers/src/main/java/org/apache/beam/it/truthmatchers}/ArtifactAsserts.java
(79%)
rename
it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/matchers
=>
truthmatchers/src/main/java/org/apache/beam/it/truthmatchers}/ArtifactsSubject.java
(87%)
copy sdks/java/ml/inference/{openai => gemini}/build.gradle (68%)
copy
examples/multi-language/src/main/java/org/apache/beam/examples/multilanguage/JavaPrefixConfiguration.java
=>
sdks/java/ml/inference/gemini/src/main/java/org/apache/beam/sdk/ml/inference/gemini/GeminiImageResponse.java
(66%)
create mode 100644
sdks/java/ml/inference/gemini/src/main/java/org/apache/beam/sdk/ml/inference/gemini/GeminiInferenceFunctions.java
create mode 100644
sdks/java/ml/inference/gemini/src/main/java/org/apache/beam/sdk/ml/inference/gemini/GeminiModelHandler.java
create mode 100644
sdks/java/ml/inference/gemini/src/main/java/org/apache/beam/sdk/ml/inference/gemini/GeminiModelParameters.java
copy
runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/TestSource.java
=>
sdks/java/ml/inference/gemini/src/main/java/org/apache/beam/sdk/ml/inference/gemini/GeminiRequestFunction.java
(62%)
copy
examples/multi-language/src/main/java/org/apache/beam/examples/multilanguage/JavaPrefixConfiguration.java
=>
sdks/java/ml/inference/gemini/src/main/java/org/apache/beam/sdk/ml/inference/gemini/GeminiStringInput.java
(70%)
copy
examples/multi-language/src/main/java/org/apache/beam/examples/multilanguage/JavaPrefixConfiguration.java
=>
sdks/java/ml/inference/gemini/src/main/java/org/apache/beam/sdk/ml/inference/gemini/GeminiStringResponse.java
(69%)
copy
{examples/java/cdap/hubspot/src/main/java/org/apache/beam/examples/complete/cdap/hubspot/options
=>
sdks/java/ml/inference/gemini/src/main/java/org/apache/beam/sdk/ml/inference/gemini}/package-info.java
(89%)
create mode 100644
sdks/java/ml/inference/gemini/src/test/java/org/apache/beam/sdk/ml/inference/gemini/GeminiModelHandlerIT.java
create mode 100644
sdks/java/ml/inference/gemini/src/test/java/org/apache/beam/sdk/ml/inference/gemini/GeminiModelHandlerTest.java
create mode 100644
sdks/java/ml/inference/remote/src/main/java/org/apache/beam/sdk/ml/inference/remote/PredictionResultCoder.java
create mode 100644
website/www/site/content/en/blog/beam-summit-2026-interview-with-raj-katakam.md
create mode 100644 website/www/site/content/en/performance/icebergio/_index.md
