This is an automated email from the ASF dual-hosted git repository.
github-actions[bot] pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from 4d975e5ec60 Avoid RequestResponseIO throttler access when throttler is
absent (#39112)
add 904f8898ab2 Fix PostCommit Java DataflowV2 workflow (#39232)
add 4bf5a6afb50 Bump github.com/aws/aws-sdk-go-v2/service/s3 in /sdks
(#39227)
add 4cde7ed933a Bump github.com/aws/aws-sdk-go-v2/config in /sdks (#39229)
add 7502ae2d529 Bump golang.org/x/text from 0.38.0 to 0.39.0 in /sdks
(#39230)
add f35fa826f44 Bump github.com/aws/aws-sdk-go-v2/feature/s3/manager in
/sdks (#39231)
add bdc26675c91 Drop envoy-data-plane/betterproto dependency from Python
SDK (#39213)
add c6e13b60fa4 Revert "Fix Python VarIntCoder OverflowError on uint64
values (#39047)" (#39218)
add 20494736bb9 Add MLTransform embedding benchmarks (#38917)
add d1c1a681d83 [aws2] Make SqsIOWriteBatchesTest timeout tests
timing-independent (#38971)
add 4a82e692e78 remove todos etc with pubsublite deprecation (#39237)
add 123dc7feaa4 Bump CI Java version to Java21 (#39205)
No new revisions were added by this update.
Summary of changes:
.../actions/setup-environment-action/action.yml | 2 +-
.../arc/images/Dockerfile | 18 +-
.github/workflows/README.md | 2 +-
.../beam_Inference_Python_Benchmarks_Dataflow.yml | 36 +++
.../beam_PerformanceTests_xlang_KafkaIO_Python.yml | 6 +-
.../workflows/beam_PostCommit_Java_DataflowV2.yml | 4 +-
.../beam_PostCommit_Java_Jpms_Dataflow.yml | 2 -
.../workflows/beam_PostCommit_Java_Jpms_Direct.yml | 2 -
.../beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml | 7 +-
.../beam_PostCommit_Python_Xlang_Gcp_Direct.yml | 6 +-
.../beam_PostCommit_Python_Xlang_IO_Dataflow.yml | 7 +-
.../beam_PostCommit_Python_Xlang_IO_Direct.yml | 7 +-
.../beam_PostCommit_TransformService_Direct.yml | 9 +-
.github/workflows/beam_PostCommit_XVR_Direct.yml | 5 +-
.github/workflows/beam_PostCommit_XVR_Flink.yml | 5 +-
.../beam_PostCommit_XVR_GoUsingJava_Dataflow.yml | 5 +-
...eam_PostCommit_XVR_JavaUsingPython_Dataflow.yml | 7 +-
..._PostCommit_XVR_PythonUsingJavaSQL_Dataflow.yml | 7 +-
...eam_PostCommit_XVR_PythonUsingJava_Dataflow.yml | 7 +-
.github/workflows/beam_PostCommit_XVR_Spark3.yml | 5 +-
.../beam_PreCommit_Java_Cassandra_IO_Direct.yml | 2 +
.../beam_PreCommit_Java_Debezium_IO_Direct.yml | 12 +-
...am_PreCommit_Java_Examples_Dataflow_Java11.yml} | 18 +-
.../beam_PreCommit_Java_Flink_Versions.yml | 2 +-
.../beam_PreCommit_Java_HCatalog_IO_Direct.yml | 2 +
.../workflows/beam_PreCommit_Java_IOs_Direct.yml | 13 +
.../beam_PreCommit_Java_Spark_Versions.yml | 4 +-
.../workflows/beam_PreCommit_Kotlin_Examples.yml | 8 +-
.../workflows/beam_Publish_Beam_SDK_Snapshots.yml | 5 +-
...flow_MLTransform_Image_Embedding_CPU_Batch.txt} | 38 +--
...flow_MLTransform_Image_Embedding_GPU_Batch.txt} | 40 ++--
..._Dataflow_MLTransform_Text_Embedding_Batch.txt} | 40 ++--
.test-infra/tools/refresh_looker_metrics.py | 5 +-
CHANGES.md | 1 +
build.gradle.kts | 7 +-
examples/java/build.gradle | 7 +
examples/kotlin/build.gradle | 7 +
infra/iam/roles/roles_config.yaml | 2 +-
.../runners/fnexecution/ServerFactoryTest.java | 19 +-
runners/spark/spark_runner.gradle | 5 +-
sdks/go.mod | 24 +-
sdks/go.sum | 48 ++--
.../java/org/apache/beam/sdk/util/ApiSurface.java | 7 +-
.../sdk/options/PipelineOptionsFactoryTest.java | 56 +++--
.../providers/JavaFilterTransformProviderTest.java | 8 +-
.../beam/sdk/util/common/ReflectHelpersTest.java | 15 +-
sdks/java/extensions/arrow/build.gradle | 7 +-
sdks/java/extensions/avro/build.gradle | 7 +
.../sdk/io/aws2/sqs/SqsIOWriteBatchesTest.java | 97 ++++++--
sdks/java/io/cassandra/build.gradle | 21 ++
sdks/java/io/cdap/build.gradle | 5 +
.../org/apache/beam/sdk/io/csv/CsvIOParseTest.java | 6 +-
sdks/java/io/influxdb/build.gradle | 10 +-
.../apache/beam/sdk/io/influxdb/InfluxDbIO.java | 22 ++
.../beam/sdk/io/influxdb/InfluxDbIOTest.java | 104 ++++----
sdks/java/io/kafka/build.gradle | 1 +
.../singlestore/SingleStoreDefaultRowMapper.java | 3 +-
sdks/python/.yapfignore | 1 +
sdks/python/apache_beam/coders/coder_impl.pxd | 1 +
sdks/python/apache_beam/coders/coder_impl.py | 34 +--
.../apache_beam/coders/coders_test_common.py | 18 --
.../ml_transform/mltransform_image_embedding.py | 263 +++++++++++++++++++++
.../mltransform_image_embedding_test.py | 105 ++++++++
.../ml_transform/mltransform_text_embedding.py | 186 +++++++++++++++
.../mltransform_text_embedding_test.py | 94 ++++++++
.../apache_beam/io/components/rate_limit.proto | 64 +++++
.../apache_beam/io/components/rate_limit_pb2.py | 63 +++++
.../apache_beam/io/components/rate_limiter.py | 39 ++-
.../apache_beam/io/components/rate_limiter_test.py | 62 ++++-
.../mltransform_embedding_tests_requirements.txt} | 5 +-
.../mltransform_image_embedding_benchmark.py | 128 ++++++++++
.../mltransform_one_hot_encoding_benchmark.py | 79 ++-----
.../mltransform_text_embedding_benchmark.py | 117 +++++++++
.../testing/load_tests/dataflow_cost_benchmark.py | 143 ++++++++++-
.../container/ml/py310/base_image_requirements.txt | 2 -
.../container/ml/py310/gpu_image_requirements.txt | 2 -
.../container/ml/py311/base_image_requirements.txt | 2 -
.../container/ml/py311/gpu_image_requirements.txt | 2 -
.../container/ml/py312/base_image_requirements.txt | 2 -
.../container/ml/py312/gpu_image_requirements.txt | 2 -
.../container/ml/py313/base_image_requirements.txt | 2 -
.../container/py310/base_image_requirements.txt | 2 -
.../container/py311/base_image_requirements.txt | 2 -
.../container/py312/base_image_requirements.txt | 2 -
.../container/py313/base_image_requirements.txt | 2 -
.../container/py314/base_image_requirements.txt | 2 -
sdks/python/pyproject.toml | 1 +
sdks/python/scripts/run_lint.sh | 1 +
sdks/python/setup.py | 4 -
.../dsls/sql/extensions/create-external-table.md | 52 ----
website/www/site/content/en/performance/_index.md | 3 +
.../mltransform-image-embedding-cpu/_index.md | 48 ++++
.../mltransform-image-embedding-gpu/_index.md | 48 ++++
.../_index.md | 24 +-
website/www/site/data/performance.yaml | 49 +++-
95 files changed, 1929 insertions(+), 554 deletions(-)
rename .github/workflows/{beam_PreCommit_Java_Examples_Dataflow_Java21.yml =>
beam_PreCommit_Java_Examples_Dataflow_Java11.yml} (92%)
copy
.github/workflows/load-tests-pipeline-options/{beam_Inference_Python_Benchmarks_Dataflow_Table_Row_Inference_Stream.txt
=>
beam_Inference_Python_Benchmarks_Dataflow_MLTransform_Image_Embedding_CPU_Batch.txt}
(59%)
copy
.github/workflows/load-tests-pipeline-options/{beam_Inference_Python_Benchmarks_Dataflow_Table_Row_Inference_Stream.txt
=>
beam_Inference_Python_Benchmarks_Dataflow_MLTransform_Image_Embedding_GPU_Batch.txt}
(54%)
copy
.github/workflows/load-tests-pipeline-options/{beam_Inference_Python_Benchmarks_Dataflow_MLTransform_Generate_Vocab_Batch.txt
=>
beam_Inference_Python_Benchmarks_Dataflow_MLTransform_Text_Embedding_Batch.txt}
(62%)
create mode 100644
sdks/python/apache_beam/examples/ml_transform/mltransform_image_embedding.py
create mode 100644
sdks/python/apache_beam/examples/ml_transform/mltransform_image_embedding_test.py
create mode 100644
sdks/python/apache_beam/examples/ml_transform/mltransform_text_embedding.py
create mode 100644
sdks/python/apache_beam/examples/ml_transform/mltransform_text_embedding_test.py
create mode 100644 sdks/python/apache_beam/io/components/rate_limit.proto
create mode 100644 sdks/python/apache_beam/io/components/rate_limit_pb2.py
copy sdks/python/{.coveragerc =>
apache_beam/ml/transforms/mltransform_embedding_tests_requirements.txt} (91%)
create mode 100644
sdks/python/apache_beam/testing/benchmarks/inference/mltransform_image_embedding_benchmark.py
create mode 100644
sdks/python/apache_beam/testing/benchmarks/inference/mltransform_text_embedding_benchmark.py
create mode 100644
website/www/site/content/en/performance/mltransform-image-embedding-cpu/_index.md
create mode 100644
website/www/site/content/en/performance/mltransform-image-embedding-gpu/_index.md
copy website/www/site/content/en/performance/{mltransformonehot =>
mltransform-text-embedding}/_index.md (55%)
