This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from e809912fa30 Fix a typo in the GenAI notebook (#27418)
add 88e635a2b0a [Python] Make BigQuery load job names deterministic for
retry resilience (#27384)
add dcd8765da76 Support for read from Cosmos DB Core SQL API (#25672)
add cc4b8f0782c fix SparkRunner url (#27405)
add fcde1d15b29 Bump cloud.google.com/go/datastore from 1.12.0 to 1.12.1
in /sdks (#27422)
add 8140ccd41cd Beam pre commit java examples dataflow (#27432)
add e8e604d1c51 Add build.gradle.kts to whitespace check (#27431)
add 738c02cac4d Runner image buildx and Website Precommit update (#27427)
add 9f8b0e3f9f9 Bump cloud.google.com/go/bigtable from 1.18.1 to 1.19.0 in
/sdks (#27421)
add 5a5779fe43f Refactor MLTrasform BaseOperation class (#27389)
add c5d6183cee1 Implement environment variable type (#27407)
add ca674aad086 Remove unneeded Dataflow Runner v1 code. (#27196)
add a0004b33e56 Provision a redis cluster on Kubernetes (#27409)
add 08538eb377b [AWS Sqs] Add SqsIO.writeBatches for improved performance
(closes #21429) (#26946)
add f80f500bfd4 adding main labels to workflows. Updated whitespace job
(#27442)
add 5f5c3c1e71c Add beam_PreCommit_RAT job (#27423)
add f376d1678ed Unbreak presubmit lint. (#27447)
add 0a85f0e6762 Update runinference.md (#27440)
add 8aedbc8d970 Bump github.com/tetratelabs/wazero from 1.2.1 to 1.3.0 in
/sdks (#27446)
add e5d2c2454d3 [Python] Fix python lint (#27451)
add 5cc18d46412 Add tighter bound to VertexAI retry filter (#27429)
add 6eb1b7e4dec Update document Storage API Support in `Google BigQuery
I/O connector` for Python SDK (#26889)
add a5faf34f767 Fix prober tests (#27453)
add 1ae68e00c88 Bump semver (#27452)
add 234873730f0 fix typo in docstring (#27456)
add 63d5171b70f [Python] use get_buffer to fetch buffer when the buffer is
None (#27373)
add bd59d09a9bb Do not prompt yN in gpg sign (#27455)
add 33518fb108d Add dockerhub step to release guide (#27457)
add d799365dbaf [Yaml] Unit Tests for SafeLineLoader and LightweightScope
(yaml/yaml_transform.py) (#27086)
add c9fe1586254 Remove ValidatesRunner_V1 suite (#26543)
No new revisions were added by this update.
Summary of changes:
.../arc/images/Dockerfile | 8 +-
.github/workflows/README.md | 13 +-
.github/workflows/beam_PreCommit_Go.yml | 5 +-
.../beam_PreCommit_Java_Examples_Dataflow.yml | 20 +-
...honAutoformatter.yml => beam_PreCommit_RAT.yml} | 31 +-
.github/workflows/beam_PreCommit_Typescript.yml | 2 +-
.github/workflows/beam_PreCommit_Website.yml | 79 +-
.github/workflows/beam_PreCommit_Whitespace.yml | 37 +-
.github/workflows/build_release_candidate.yml | 2 +-
.github/workflows/job_PreCommit_Python.yml | 2 +-
.../job_PreCommit_PythonAutoformatter.yml | 6 +-
.../workflows/job_PreCommit_Python_Dataframes.yml | 2 +-
.../workflows/job_PreCommit_Python_DockerBuild.yml | 2 +-
.../workflows/job_PreCommit_Python_Examples.yml | 2 +-
.github/workflows/job_PreCommit_Python_Runners.yml | 2 +-
.../workflows/job_PreCommit_Python_Transforms.yml | 2 +-
.github/workflows/job_Precommit_PythonLint.yml | 4 +-
.../workflows/job_Precommit_Python_Coverage.yml | 2 +-
.test-infra/jenkins/README.md | 1 -
...stCommit_Python_ValidatesRunner_Dataflow.groovy | 7 +-
...ommit_Python_ValidatesRunner_Dataflow_V2.groovy | 46 --
.../jenkins/job_PreCommit_Whitespace.groovy | 1 +
.../metrics/src/test/groovy/ProberTests.groovy | 2 +-
.test-infra/pipelines/go.mod | 21 +
.test-infra/pipelines/go.sum | 2 +
.../03.io/api-overuse-study/02.redis/README.md | 46 ++
.../api-overuse-study/02.redis/common.tfvars} | 13 +-
.../03.io/api-overuse-study/02.redis/data.tf} | 18 +-
.../03.io/api-overuse-study/02.redis/provider.tf} | 19 +-
.../03.io/api-overuse-study/02.redis/redis.tf} | 29 +-
.../03.io/api-overuse-study/02.redis/variables.tf} | 16 +-
.../src/main/go/internal/environment/variable.go | 81 ++
.../main/go/internal/environment/variable_test.go | 358 +++++++++
CHANGES.md | 3 +-
README.md | 2 +-
build.gradle.kts | 6 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 6 +-
learning/tour-of-beam/terraform/build.gradle.kts | 10 +-
.../src/main/scripts/build_release_candidate.sh | 4 +-
sdks/go.mod | 6 +-
sdks/go.sum | 12 +-
.../sdk/io/aws2/common/AsyncBatchWriteHandler.java | 403 ++++++++++
.../sdk/io/aws2/common/ClientConfiguration.java | 1 +
.../io/aws2/kinesis/AsyncPutRecordsHandler.java | 271 -------
.../apache/beam/sdk/io/aws2/kinesis/KinesisIO.java | 46 +-
.../org/apache/beam/sdk/io/aws2/sqs/SqsIO.java | 488 +++++++++++-
.../io/aws2/common/AsyncBatchWriteHandlerTest.java | 267 +++++++
.../aws2/kinesis/AsyncPutRecordsHandlerTest.java | 192 -----
.../sdk/io/aws2/sqs/SqsIOWriteBatchesTest.java | 264 +++++++
.../beam/sdk/io/aws2/sqs/testing/SqsIOIT.java | 31 +-
sdks/java/io/{azure => azure-cosmos}/build.gradle | 37 +-
.../apache/beam/sdk/io/azure/cosmos/CosmosIO.java | 307 ++++++++
.../beam/sdk/io/azure/cosmos/CosmosOptions.java | 81 ++
.../beam/sdk/io/azure/cosmos/NormalizedRange.java | 66 ++
.../beam/sdk/io/azure/cosmos/package-info.java | 14 +-
.../com/azure/cosmos/examples/common/Address.java | 44 +-
.../com/azure/cosmos/examples/common/Child.java | 68 ++
.../com/azure/cosmos/examples/common/Families.java | 133 ++++
.../com/azure/cosmos/examples/common/Family.java | 88 +++
.../com/azure/cosmos/examples/common/Parent.java | 42 +-
.../java/com/azure/cosmos/examples/common/Pet.java | 24 +-
.../beam/sdk/io/azure/cosmos/CosmosIOTest.java | 176 +++++
sdks/java/io/azure/README.md | 19 -
sdks/java/io/azure/build.gradle | 23 +-
.../{BlobstoreOptions.java => AzureOptions.java} | 72 +-
.../sdk/io/azure/options/BlobstoreOptions.java | 39 +-
sdks/python/apache_beam/coders/coders.py | 133 ----
.../apache_beam/coders/coders_test_common.py | 41 -
sdks/python/apache_beam/coders/row_coder.py | 9 -
sdks/python/apache_beam/coders/row_coder_test.py | 11 -
.../apache_beam/examples/snippets/snippets.py | 57 ++
.../apache_beam/examples/snippets/snippets_test.py | 10 +
.../apache_beam/io/gcp/bigquery_file_loads.py | 21 +-
.../apache_beam/io/gcp/bigquery_file_loads_test.py | 56 +-
sdks/python/apache_beam/io/gcp/bigquery_tools.py | 2 +-
sdks/python/apache_beam/io/range_trackers.py | 2 +-
.../ml/inference/vertex_ai_inference.py | 22 +-
.../ml/inference/vertex_ai_inference_test.py | 39 +
sdks/python/apache_beam/ml/transforms/base.py | 59 +-
sdks/python/apache_beam/ml/transforms/base_test.py | 2 +-
sdks/python/apache_beam/ml/transforms/handlers.py | 8 +-
.../apache_beam/ml/transforms/handlers_test.py | 13 +-
sdks/python/apache_beam/ml/transforms/tft.py | 40 +-
sdks/python/apache_beam/ml/transforms/utils.py | 3 +-
.../runners/dataflow/dataflow_runner.py | 830 +--------------------
.../runners/dataflow/dataflow_runner_test.py | 369 +--------
.../runners/dataflow/internal/apiclient.py | 95 +--
.../runners/dataflow/internal/apiclient_test.py | 53 --
.../apache_beam/runners/dataflow/internal/names.py | 76 --
.../runners/dataflow/ptransform_overrides.py | 126 ----
.../apache-beam-jupyterlab-sidepanel/yarn.lock | 24 +-
.../runners/portability/fn_api_runner/fn_runner.py | 16 +-
.../portability/fn_api_runner/fn_runner_test.py | 9 +
.../python/apache_beam/yaml/yaml_transform_test.py | 126 ++++
sdks/python/setup.py | 1 +
sdks/python/test-suites/dataflow/build.gradle | 12 -
sdks/python/test-suites/gradle.properties | 5 -
sdks/python/test-suites/tox/py38/build.gradle | 2 +
settings.gradle.kts | 1 +
.../site/content/en/contribute/release-guide.md | 3 +
.../documentation/io/built-in/google-bigquery.md | 19 +-
.../transforms/python/elementwise/runinference.md | 6 +-
102 files changed, 3707 insertions(+), 2720 deletions(-)
copy .github/workflows/{job_PreCommit_PythonAutoformatter.yml =>
beam_PreCommit_RAT.yml} (78%)
delete mode 100644
.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Dataflow_V2.groovy
create mode 100644 .test-infra/pipelines/go.mod
create mode 100644 .test-infra/pipelines/go.sum
create mode 100644
.test-infra/pipelines/infrastructure/03.io/api-overuse-study/02.redis/README.md
copy .test-infra/{jenkins/job_PreCommit_Whitespace.groovy =>
pipelines/infrastructure/03.io/api-overuse-study/02.redis/common.tfvars} (75%)
copy .test-infra/{jenkins/job_PreCommit_Whitespace.groovy =>
pipelines/infrastructure/03.io/api-overuse-study/02.redis/data.tf} (75%)
copy .test-infra/{jenkins/job_PreCommit_Whitespace.groovy =>
pipelines/infrastructure/03.io/api-overuse-study/02.redis/provider.tf} (75%)
copy .test-infra/{jenkins/job_PreCommit_Whitespace.groovy =>
pipelines/infrastructure/03.io/api-overuse-study/02.redis/redis.tf} (65%)
copy .test-infra/{jenkins/job_PreCommit_Whitespace.groovy =>
pipelines/infrastructure/03.io/api-overuse-study/02.redis/variables.tf} (75%)
create mode 100644
.test-infra/pipelines/src/main/go/internal/environment/variable.go
create mode 100644
.test-infra/pipelines/src/main/go/internal/environment/variable_test.go
create mode 100644
sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/AsyncBatchWriteHandler.java
delete mode 100644
sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/AsyncPutRecordsHandler.java
create mode 100644
sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/common/AsyncBatchWriteHandlerTest.java
delete mode 100644
sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/AsyncPutRecordsHandlerTest.java
create mode 100644
sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/sqs/SqsIOWriteBatchesTest.java
copy sdks/java/io/{azure => azure-cosmos}/build.gradle (60%)
create mode 100644
sdks/java/io/azure-cosmos/src/main/java/org/apache/beam/sdk/io/azure/cosmos/CosmosIO.java
create mode 100644
sdks/java/io/azure-cosmos/src/main/java/org/apache/beam/sdk/io/azure/cosmos/CosmosOptions.java
create mode 100644
sdks/java/io/azure-cosmos/src/main/java/org/apache/beam/sdk/io/azure/cosmos/NormalizedRange.java
copy .test-infra/jenkins/job_PreCommit_Whitespace.groovy =>
sdks/java/io/azure-cosmos/src/main/java/org/apache/beam/sdk/io/azure/cosmos/package-info.java
(75%)
copy .test-infra/jenkins/job_PreCommit_Whitespace.groovy =>
sdks/java/io/azure-cosmos/src/test/java/com/azure/cosmos/examples/common/Address.java
(58%)
create mode 100644
sdks/java/io/azure-cosmos/src/test/java/com/azure/cosmos/examples/common/Child.java
create mode 100644
sdks/java/io/azure-cosmos/src/test/java/com/azure/cosmos/examples/common/Families.java
create mode 100644
sdks/java/io/azure-cosmos/src/test/java/com/azure/cosmos/examples/common/Family.java
copy .test-infra/jenkins/job_PreCommit_Whitespace.groovy =>
sdks/java/io/azure-cosmos/src/test/java/com/azure/cosmos/examples/common/Parent.java
(58%)
copy .test-infra/jenkins/job_PreCommit_Whitespace.groovy =>
sdks/java/io/azure-cosmos/src/test/java/com/azure/cosmos/examples/common/Pet.java
(73%)
create mode 100644
sdks/java/io/azure-cosmos/src/test/java/org/apache/beam/sdk/io/azure/cosmos/CosmosIOTest.java
delete mode 100644 sdks/java/io/azure/README.md
copy
sdks/java/io/azure/src/main/java/org/apache/beam/sdk/io/azure/options/{BlobstoreOptions.java
=> AzureOptions.java} (51%)
create mode 100644
sdks/python/apache_beam/ml/inference/vertex_ai_inference_test.py
