This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch
dependabot/go_modules/sdks/google.golang.org/protobuf-1.32.0
in repository https://gitbox.apache.org/repos/asf/beam.git
omit 3e8ca5012dd Bump google.golang.org/protobuf from 1.31.0 to 1.32.0 in
/sdks
add e6e180b6896 Switch to use self hosted runners for linux unit tests
(#29907)
add 02e26c1828f raise error if transforms are provided while read mode in
MLTransform (#29909)
add a5f0cda4f3b Fix: creating templates with the Typescript SDK (#29887)
add 308223f447f [release] 2.53.0 Website Updates (#29856)
add 24e5f3a6c01 [release] Add extra context for release branch cut action
input (#29920)
add f88c78b79ca [release] Update Release Guide Instructions (#29919)
add 8bd4a1e525b [Typescript] stop service when done (#29929)
add c6f0d9c462a Add notebooks for text embeddings (#29893)
add a6948ad71e3 Install beam from pypi instead of github in mltransform
notebooks (#29934)
add b9ae59a9978 Remove TODO in the notebook (#29935)
add 0b0d97325c1 Update pmc finalization in release-guide.md (#29936)
add f79eaddf574 Preparing the data processing notebooks for import to
devsite (#29937)
add 3260a7bf25a Final edit on data preprocessing notebooks (#29940)
add 4cf1b15024d Updated the doc with Security Fixes for 2.53.0 (#29941)
add 66f98a0670c Update finalize_release.yml (#29918)
add 75cfbee1591 Update MLTransform docs (#29910)
add c9e036e40e4 [Python BQ] Retry get_table for quota errors (#28820)
add 992e4fd5019 upgrade prettier
add 63503391562 Merge pull request #29942 [Typescript] upgrade prettier
add 094eb7dc5d7 Increase granularity of DICOM IO Unit Tests (#29956)
add b666c6434dc [Dataflow Streaming] Make SideInputCache bytes and expiry
configurable (#29871)
add 97ca4430d97 remove threadlocal for DESERIALIZATION_CONTEXT and use
copy instead (#29952)
add f311d327aac Bump org.checkerframework:checkerframework-gradle-plugin
add bb0a2037970 Merge pull request #29811: Bump
org.checkerframework:checkerframework-gradle-plugin from 0.6.34 to 0.6.37
add 8fb06da4ce8 [RRIO] Build RequestResponseIO and related dependency
changes (#29710)
add 5e8fef44939 Bump timeout to 120 minutes (#29964)
add 8aa16df9bd1 Bump runner v2 beam-master to 20240109 (#29965)
add 6066af3dbd5 [YAML] - Kafka Proto String schema (#29835)
add a3e065366e9 Bump GCP Libraries BOM to v26.29.0 (#29922)
add 2eb375be82f Downgrade artifact version in build_wheels (#29960)
add 0a368056904 Add support for pandas 2.0 (#28636)
add ee3d0a3c5ca [YAML] Add AssignTimestamps transform. (#29969)
add eb7addc1a5b [yaml] Replace pipeline_spec with yaml_pipeline in main.py
add 0c955dfdb7e Merge pull request #29916 [YAML] Replace pipeline_spec
with yaml_pipeline in main.py
add a80522ef5e7 update dependencies (#29974)
add 0bc1556d5ed Allows Java SDK to pass through external coders during
proto translation without fully interpretting them (#29859)
add 9697ae818d2 Update container tag (#29977)
add ba73001d687 Feature/bigtable error handling (#29885)
add efd50ede0dd update CHANGES.md with error handling for fileIO and
bigtableIO (#29983)
add 96d092b31ec Only auth and push on master branch (#29995)
add 812684f3cef [#29772][prism] Handle EventTime Timers. (#29900)
add 68849914cfc Fixes to environment cloning.
add 0fb42e5ee32 [YAML] Better support for inline PyTransforms.
add cfec51224e4 Merge pull request #29958 [YAML] Better support for inline
PyTransforms.
add d70f516ce20 fix: Spanner PostgreSQL should filter on 'public' schema
(#29984)
add db0a4deea2f [IT] Pub/Sub RM should delete subscriptions that were
created by Beam (#29957)
add c38dc77283f Upgrade Parquet to 0.13.1 (#29953)
add b0d74c3c6f3 Generate per transform schema for validation.
add 48650f12894 Conditional validation.
add e50b7cc89d2 Conditionally validate transform schemas.
add cb825565a9d Better line numbers for messages.
add 1c7f8392c3b Fix imports.
add 7cb559f2677 Merge pull request #29830 [YAML] More complete schema
validation.
add 0d58d23f2d9 [#29180][prism] Return total element count to progress
loop. Split less aggressively. (#29968)
add 057c1d56f88 Change colab links (#30006)
add 6cd53fad82a Support DLQ in MLTransform (#29938)
add 308c38e243c [Typescript] Worker improvements (#29903)
add dd5bbb2d77d Add a read timeout and cache BigQueryIOMetadata (#29662)
add c11b7c1f7cb Update scaling-streaming-workload.md (#29921)
add cf201198df9 sickbay new reshuffle test on runners that do not support
it
add 96a9830ac27 Merge pull request #29994: sickbay new reshuffle test on
runners that do not support it
add 534015c0b8e [Typescript] creating dataflow job template fix (#29928)
add a41fd2cba03 Remove INFO from playground python logger
add 7f5c4b9b72d Merge pull request #29948 Remove INFO from playground
python logger
add c62c24dda4c [yaml] Revise yaml user label (#29989)
add fc626b8b280 typo in docs
add f1c154778f5 Merge pull request #30021: typo in docs
add 5a4cbe2074f Additional test for windmill OrderedListState
implementation. (#29970)
add d1a17fa3cbd Update `google_api_services_healthcare` [01/24] (#30027)
add 4b909912ae8 Vendor grpc 1.60.1 (#29954)
add 730f1c5edb3 [YAML] Support comment and delimiter attributes on CSV IO.
add 87145a8c2c1 Merge pull request #30013 [YAML] Support comment and
delimiter attributes on CSV IO.
add b5dc728b677 Fix an edge case of getting duplicated records when using
TextIO. (#30026)
add 3faa8729677 add #29987 to 2.53 known issues (#29988)
add 52b4a9cb58e Update to fix endpoint connectivity (#30022)
add 79b9de283fe GCS autosharding flag (#29886)
add e867ed7cdce Worker message plumbing (#29879)
add b4c23b32f2b Correct Examples of utilizing `UploadToDicomStore` (#30008)
add 65811eed6ea Upgrade checkerframework to 3.42.0
add 8772389d0c2 Merge pull request #29961: Upgrade checkerframework to
3.42.0
add ba05f0f0343 Add a enum for the new value capture type
NEW_ROW_AND_OLD_VALUES. Also add related unit tests.
add e019de89307 Merge pull request #30012 Support spanner
NEW_ROW_AND_OLD_VALUES.
add 02b369d87a7 ReadFromTsv example.
add 2ff551eeb9e Merge pull request #29997 Add a ReadFromTsv example.
add 1de8454ddd2 [Python] Enrichment Transform with BigTable handler
(#30001)
add 0bfd1a47ff4 Update to libraries-bom 26.30.0 (#30028)
add 81688eb57b3 [YAML] add time unit suffix to windowing (#30035)
add e594f57a2d7 Undeprecate reshuffle (#30049)
add 195d8dcb367 Add logging to help understand errors from #28131. (#30052)
add ef8d3faa7df Switch back to use official jamm (#30034)
add b4be68e2404 [Python BQ] Substitute final destination schema when no
input schema is specified (#30015)
add 23e83092345 Fix buildx always been true (#30051)
add 774082f57a9 Exclude newly added failing validation runner tests
(#30030)
add d5a7fc92cdf Centralize exec stage urn string (#29947)
add e4b81809c13 fix config translator. If config is set on BigtableIO
directly, it should override configs in BigtableOptions (#30039)
add 175346e3507 Automatically enable Dataflow Runner v2 for pipeliens that
use the transform upgrade feature
add c0cc7c4db1b Merge pull request #30061: Automatically enable Dataflow
Runner v2 for pipeliens that use the transform upgrade feature
add 5eeffc7715f Address Feedback from Beam Website Feedback Form. (#30064)
add d014a98936e Time at max threads fix (#30041)
add 173d8340b20 split write rename and temp table deletion into to dofns
with a shuffle (#30023)
add 821a1694c5c Fix Cassandra Test (#30075)
add 5e7edc45598 Heartbeats (#29963)
add 0a813b99657 Plumbing remaining autoscaling metrics (#30070)
add 93339bcf061 Fixes an upgrade compatibility breakage for the BQ write
transform (#30032)
add e8e0ccfdd7f Bump pillow (#30076)
add cffeff78f62 Fix Dicom IO Integration Tests (#30073)
add 58650378fe7 [flink] Fix watermark generation for empty UnboundedSource
(#30050)
add 90e7c9e8f6b Merge pull request #30066: [flink] Fix watermark
generation for empty UnboundedSource
add 77285589fe3 Use vendored grpc 1.60.1 (#30069)
add 0b06cd897d2 Fix BigQuerySinkMetrics constants and increment metrics in
more places. (#30067)
add 0c2b1a512db Add converter class to convert metrics to
PerStepNamespaceMetrics (#30084)
add 9e5b112be9f Use FileSystems.exist to check the remote the path (#30082)
add 405e6c35bb8 [YAML] Use default logging level for main.py
add 330a632fefb Merge pull request #30087 [YAML] Use default logging level
for main.py
add 9dd30765476 Bump pillow from 10.0.1 to 10.2.0 in
/sdks/python/apache_beam/examples/ml-orchestration/kfp/components/preprocessing
(#30071)
add 7ff25d89625 [YAML] Allow windowing to be done in Java or Python.
(#30055)
add b22fe9287b0 [YAML] LogForTesting improvements. (#30005)
add 0eefb797765 [YAML] Expose flatten implementation from Java. (#30057)
add 8f3887bb512 [YAML] Require numStreams for unbounded
BigQueryStorageWriteApiSchemaTransform xlang transform
add 11646ca7da2 Merge pull request #30059 [YAML] Require numStreams for
unbounded BigQueryStorageWrite
add 582d5c03569 fix for #30078 (#30079)
add 98a86b11cc7 [MLTransform]Throw an error when specifiend columns are
not present in the input data (#30085)
add f7da756ac6d Update container build version to Go 1.21.6 (#30090)
add e85d070a32b Ignore DicomMetadataRead tests (#30096)
add 27214149d34 Support dynamic destinations with Python Storage API
(#30045)
add 674fe77df47 [Python] Add an option to retain timestamp returned from
the BigTable row (#30088)
add e3f45249620 Moving to 2.55.0-SNAPSHOT on master branch.
add d759f970139 Add StreamingStepMetricsContainer::extractPerWorkerMetrics
method. (#30089)
add d62ae0144cc Add section for 2.55.0, cleanup 2.54.0 due to cut. (#30101)
add 61a62e19c4a rename ExternalSchemaTransform to ExternalTransform
(#30102)
add 93073e74e11 Decommission all remaining jenkins jobs (#30113)
add 772cf0a5c01 Don't run DicomIO Integration Tests on GCP DF Runner
(#30103)
add 3ae851876e2 Fix failing python BQ test (#30099)
add 6b6fc98ec93 Don't depend on the go 1.21 standard lib yet (#30120)
add 445151694fe Check the issue number type (#30127)
add 3fd7bf29dcd Bump Dataflow dev container version (#30124)
add 1ab504c34a6 Update update_python_dependencies.yml with all python
versions (#30126)
add 4981c0c3abc Upgrade numpy dependency (#29645)
add 1d4413d8e95 Update project root documentations reflecting Jenkins
shutdown (#30130)
add e0e20a10e99 [#28142][Go SDK] Evict closed readers from the cache.
(#30119)
add 8485ab40e12 Allow JSON type in TableSchema for BigQuery FILE_LOAD
(#29923)
add 3172736aaa3 Update Python Dependencies (#30140)
add dde81384ef2 Fix clean up stale container workflow (#30132)
add c9135e00041 Add info on getting started contributing (#30143)
add b72eacfba37 make getProcessingDistributionsForWorkId threadsafe
(#29979)
add c1c81b36187 Respect gcloud active configuration, if any, rather than
always using "default."
add ee4f8cb33f9 Merge pull request #30134 Respect gcloud active
configuration.
add 847a8f4486e [YAML] Improve docs on create.
add 8521d2768e0 [YAML] Better document naming parameter.
add ba29583d46c [YAML] Move pipeline running instructions to the top.
add e70a28acbd4 [YAML] Link to additonal resources.
add b85e1ebaea3 Some clarification on the mapping docs.
add 09aacf1003c Another note useful for trying things out.
add 799c1041473 Avoid SQL keywords for SQL table names.
add afd8b79e8e0 Clarification on aggregating transform based on user
feedback.
add 88bf2fc94fe Update sdks/python/apache_beam/yaml/yaml_provider.py
add b835111d66f Make note about requiring Graphviz for rendering pipelines.
add 3970b618371 Merge branch 'master' into yaml-create-doc
add b5dc54da26f Also mention JavaScript.
add e199e42a341 Fix whitespace in create suggestion.
add ce28c33940c Merge pull request #30007 [YAML] Several improvements to
the documentation.
add f9029616a91 Update environments.md (#30149)
add 097e6e27bfe Bump actions/cache from 3 to 4 (#30033)
add f9c9788815e Update 02_io_pubsub.md (#29986)
add 5a9a27cbafa Volatile heartbeat (#30148)
add b9fd39cb16b Fix Dataproc cleanup race condition (#30154)
add 22fefebacab BigQuery: Decouple clustering from time partitioning when
writing (#30094)
add 415b1658f50 Merge pull request #30137: Fix a misspelling in zetasql
overview page
add 1df9d6bba91 Bumping timeout for beam_PostCommit_Java_DataflowV2
(#30160)
add 4c5ee349f9b [sdks-java-core] Test for Reiterator outside loop in
CoGbkResult
add 36cab4ffe76 Merge pull request #30163: [sdks-java-core] Test for
Reiterator outside loop in CoGbkResult
add 41dee464db4 Make ReadFromBigQueryRequest id more randomized (#30156)
add 81513f10d60 Bump github.com/aws/aws-sdk-go-v2/credentials in /sdks
(#30162)
add 89d1c06e1ea Fix Fhir IO Search IT (#30158)
add a221f98a5f4 fix: fix error handling in segment reader and add an
integration test (#30147)
add ddb93272ab6 Fix git trusted directory errors in website build. (#30171)
add 30917ff7b3e Don't use '-' as a delimiter in BigQuerySinkMetrics
(#30118)
add fb0762352c2 Periodically send perWorkerMetrics on the WorkerMessages
API (#30135)
add 4f09e62f520 Improve varint encoding throughput with unrolled loop
(#29689)
add 5188e37ed44 Bump github.com/opencontainers/runc from 1.1.5 to 1.1.12
in /sdks (#30175)
add 2c8795eacb8 Bump google.golang.org/protobuf from 1.31.0 to 1.32.0 in
/sdks
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (3e8ca5012dd)
\
N -- N -- N
refs/heads/dependabot/go_modules/sdks/google.golang.org/protobuf-1.32.0
(2c8795eacb8)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.asf.yaml | 1 +
.../trigger_files/beam_PostCommit_Go_VR_Spark.json | 0
...o.json => beam_PostCommit_Java_DataflowV2.json} | 0
...o_VR_Flink.json => beam_PostCommit_Python.json} | 0
...beam_PostCommit_Python_Xlang_Gcp_Dataflow.json} | 0
.../beam_PostCommit_Python_Xlang_Gcp_Direct.json | 1 +
.github/workflows/README.md | 1 +
.github/workflows/beam_CancelStaleDataflowJobs.yml | 2 +
.../workflows/beam_CleanUpDataprocResources.yml | 2 +-
.github/workflows/beam_CleanUpGCPResources.yml | 2 +
.../workflows/beam_CleanUpPrebuiltSDKImages.yml | 8 +-
.../workflows/beam_PostCommit_Java_DataflowV2.yml | 4 +-
.../beam_PreCommit_Java_Kafka_IO_Direct.yml | 4 +-
.github/workflows/build_runner_image.yml | 4 +-
.github/workflows/build_wheels.yml | 30 +-
.github/workflows/cut_release_branch.yml | 2 +-
.github/workflows/finalize_release.yml | 17 +-
.github/workflows/go_tests.yml | 2 +-
.github/workflows/java_tests.yml | 8 +-
.github/workflows/playground_frontend_test.yml | 2 +-
.github/workflows/python_tests.yml | 8 +-
.github/workflows/tour_of_beam_frontend_test.yml | 2 +-
.github/workflows/update_python_dependencies.yml | 15 +-
.test-infra/dataproc/cleanup.sh | 4 +-
.test-infra/jenkins/README.md | 149 +-----
.test-infra/jenkins/job_00_seed.groovy | 115 -----
.test-infra/jenkins/job_Inventory.groovy | 83 ---
...mmit_CrossLanguageValidatesRunner_Direct.groovy | 51 --
...uageValidatesRunner_GoUsingJava_Dataflow.groovy | 44 --
...ValidatesRunner_PythonUsingJava_Dataflow.groovy | 49 --
.../jenkins/job_PostCommit_Java_DataflowV2.groovy | 48 --
.../job_PostCommit_Java_InfluxDBIO_IT.groovy | 55 --
...ob_PostCommit_Java_ValidatesRunner_Flink.groovy | 45 --
...Commit_Java_ValidatesRunner_Flink_Java11.groovy | 61 ---
.test-infra/jenkins/job_PostCommit_Python.groovy | 50 --
...tCommit_Python_CrossLanguage_IO_Dataflow.groovy | 55 --
.../job_PostCommit_Python_Examples_Direct.groovy | 43 --
.../job_PostCommit_Python_Examples_Flink.groovy | 43 --
..._PostCommit_Python_ValidatesRunner_Flink.groovy | 38 --
..._PostCommit_Python_ValidatesRunner_Spark.groovy | 43 --
.../jenkins/job_PostRelease_NightlySnapshot.groovy | 59 ---
.test-infra/jenkins/job_seed_standalone.groovy | 115 -----
.../google-kubernetes-engine/README.md | 14 +-
.../google-kubernetes-engine/cluster.tf | 19 +-
...vars => us-central1.apache-beam-testing.tfvars} | 4 +-
....tfvars => us-west1.apache-beam-testing.tfvars} | 2 +-
.../tools/stale_dataflow_prebuilt_image_cleaner.sh | 72 ++-
CHANGES.md | 58 ++-
CI.md | 37 +-
README.md | 3 -
buildSrc/build.gradle.kts | 2 +-
.../org/apache/beam/gradle/BeamDockerPlugin.groovy | 3 +
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 44 +-
...g_1_54_0.groovy => GrpcVendoring_1_60_1.groovy} | 96 ++--
.../org/apache/beam/gradle/VendorJavaPlugin.groovy | 9 +-
contributor-docs/release-guide.md | 17 +-
examples/java/build.gradle | 2 +-
.../transforms/DataProtectors.java | 6 +-
.../kafkatopubsub/kafka/consumer/Utils.java | 4 +-
examples/notebooks/beam-ml/README.md | 19 +-
.../beam-ml/automatic_model_refresh.ipynb | 2 +-
.../compute_and_apply_vocab.ipynb | 48 +-
.../huggingface_text_embeddings.ipynb | 399 ++++++++++++++
.../scale_data.ipynb | 62 ++-
.../vertex_ai_text_embeddings.ipynb | 308 +++++++++++
gradle.properties | 4 +-
.../it/gcp/dataflow/AbstractPipelineLauncher.java | 17 +-
.../beam/it/gcp/pubsub/PubsubResourceManager.java | 14 +-
.../it/gcp/pubsub/PubsubResourceManagerTest.java | 28 +-
learning/prompts/code-explaination/02_io_pubsub.md | 8 +-
.../beam/model/pipeline/v1/beam_runner_api.proto | 4 +
playground/infrastructure/logger.py | 2 +-
runners/core-construction-java/build.gradle | 2 +-
.../beam/runners/core/construction/BeamUrns.java | 2 +-
.../core/construction/CoderTranslation.java | 32 +-
.../core/construction/CoderTranslators.java | 2 +-
.../core/construction/CombineTranslation.java | 2 +-
.../CreatePCollectionViewTranslation.java | 2 +-
.../DefaultExpansionServiceClientFactory.java | 2 +-
.../core/construction/DisplayDataTranslation.java | 2 +-
.../runners/core/construction/Environments.java | 4 +-
.../beam/runners/core/construction/External.java | 6 +-
.../runners/core/construction/ModelCoders.java | 2 +-
.../construction/PCollectionViewTranslation.java | 2 +-
.../core/construction/PTransformTranslation.java | 5 +-
.../core/construction/ParDoTranslation.java | 4 +-
.../construction/PipelineOptionsTranslation.java | 6 +-
.../core/construction/PipelineTranslation.java | 17 +-
.../runners/core/construction/ReadTranslation.java | 4 +-
.../runners/core/construction/SdkComponents.java | 2 +-
.../core/construction/TestStreamTranslation.java | 2 +-
.../core/construction/TransformUpgrader.java | 55 +-
.../core/construction/UnknownCoderWrapper.java | 69 +++
.../core/construction/WindowIntoTranslation.java | 2 +-
.../construction/WindowingStrategyTranslation.java | 8 +-
.../core/construction/WriteFilesTranslation.java | 25 +-
.../graph/GreedyPCollectionFusers.java | 2 +-
.../core/construction/graph/QueryablePipeline.java | 2 +-
.../core/construction/BaseExternalTest.java | 6 +-
.../core/construction/CoderTranslationTest.java | 1 +
.../PipelineOptionsTranslationTest.java | 6 +-
.../core/construction/PipelineTranslationTest.java | 2 +-
.../core/construction/TransformUpgraderTest.java | 9 +-
.../construction/WindowIntoTranslationTest.java | 2 +-
.../construction/WriteFilesTranslationTest.java | 6 +-
.../construction/graph/ProtoOverridesTest.java | 2 +-
runners/core-java/build.gradle | 2 +-
.../core/metrics/ExecutionStateSampler.java | 3 +-
.../runners/core/metrics/MetricsContainerImpl.java | 2 +-
.../core/metrics/MetricsContainerStepMap.java | 6 +-
.../core/metrics/MonitoringInfoEncodings.java | 2 +-
.../beam/runners/core/metrics/ShortIdMap.java | 2 +-
.../runners/core/metrics/SimpleExecutionState.java | 2 +-
.../runners/core/metrics/SimpleStateRegistry.java | 2 +-
.../core/metrics/MonitoringInfoEncodingsTest.java | 2 +-
runners/direct-java/build.gradle | 4 +-
runners/flink/flink_runner.gradle | 2 +-
.../flink_job_server_container.gradle | 5 +-
.../FlinkBatchPortablePipelineTranslator.java | 2 +-
.../apache/beam/runners/flink/FlinkJobInvoker.java | 2 +-
.../beam/runners/flink/FlinkPipelineRunner.java | 2 +-
.../FlinkStreamingPortablePipelineTranslator.java | 2 +-
.../apache/beam/runners/flink/metrics/Metrics.java | 3 +-
.../utils/FlinkPortableRunnerUtils.java | 2 +-
.../streaming/ExecutableStageDoFnOperator.java | 4 +-
.../wrappers/streaming/FlinkKeyUtils.java | 2 +-
.../streaming/io/source/FlinkSourceReaderBase.java | 1 +
.../unbounded/FlinkUnboundedSourceReader.java | 13 +-
.../FlinkExecutableStageFunctionTest.java | 2 +-
.../streaming/ExecutableStageDoFnOperatorTest.java | 4 +-
.../wrappers/streaming/FlinkKeyUtilsTest.java | 2 +-
.../streaming/io/source/EmptyUnboundedSource.java | 101 ++++
.../io/source/FlinkSourceReaderTestBase.java | 8 +
.../unbounded/FlinkUnboundedSourceReaderTest.java | 33 ++
runners/google-cloud-dataflow-java/build.gradle | 8 +-
.../beam/runners/dataflow/DataflowMetrics.java | 8 +-
.../dataflow/DataflowPipelineTranslator.java | 2 +-
.../beam/runners/dataflow/DataflowRunner.java | 30 +-
.../options/DataflowPipelineDebugOptions.java | 16 +-
.../util/RowCoderCloudObjectTranslator.java | 2 +-
.../util/SchemaCoderCloudObjectTranslator.java | 2 +-
.../dataflow/DataflowPipelineTranslatorTest.java | 2 +-
.../beam/runners/dataflow/DataflowRunnerTest.java | 35 +-
.../google-cloud-dataflow-java/worker/build.gradle | 4 +-
.../runners/dataflow/worker/ByteArrayReader.java | 4 +-
.../runners/dataflow/worker/ByteStringCoder.java | 2 +-
.../worker/ChunkingShuffleBatchReader.java | 2 +-
.../worker/DataflowExecutionStateSampler.java | 9 +-
.../dataflow/worker/DataflowWorkUnitClient.java | 60 +++
.../worker/DataflowWorkerHarnessHelper.java | 2 +-
.../worker/GroupAlsoByWindowParDoFnFactory.java | 2 +-
.../worker/MetricTrackingWindmillServerStub.java | 32 +-
.../MetricsToPerStepNamespaceMetricsConverter.java | 196 +++++++
.../runners/dataflow/worker/PubsubDynamicSink.java | 2 +-
.../beam/runners/dataflow/worker/PubsubReader.java | 8 +
.../beam/runners/dataflow/worker/PubsubSink.java | 2 +-
.../dataflow/worker/StreamingDataflowWorker.java | 193 +++++--
.../worker/StreamingModeExecutionContext.java | 16 +-
.../dataflow/worker/StreamingSideInputFetcher.java | 2 +-
.../worker/StreamingStepMetricsContainer.java | 134 ++++-
.../dataflow/worker/UngroupedShuffleReader.java | 2 +-
.../dataflow/worker/UngroupedWindmillReader.java | 8 +
.../dataflow/worker/WindmillComputationKey.java | 4 +-
.../dataflow/worker/WindmillNamespacePrefix.java | 2 +-
.../beam/runners/dataflow/worker/WindmillSink.java | 2 +-
.../dataflow/worker/WindmillTimerInternals.java | 2 +-
...Prefix.java => WorkItemCancelledException.java} | 40 +-
.../runners/dataflow/worker/WorkUnitClient.java | 28 +
.../dataflow/worker/WorkerCustomSources.java | 5 +-
.../dataflow/worker/streaming/ActiveWorkState.java | 66 ++-
.../worker/streaming/ComputationState.java | 14 +-
.../dataflow/worker/streaming/ShardedKey.java | 2 +-
.../dataflow/worker/streaming/StageInfo.java | 9 +
.../runners/dataflow/worker/streaming/Work.java | 11 +
.../worker/streaming/sideinput/SideInputCache.java | 11 +-
.../streaming/sideinput/SideInputStateFetcher.java | 6 +-
.../dataflow/worker/util/BoundedQueueExecutor.java | 4 +-
.../common/worker/ByteArrayShufflePosition.java | 4 +-
.../worker/GroupingShuffleEntryIterator.java | 2 +-
.../common/worker/KeyGroupedShuffleEntries.java | 2 +-
.../worker/util/common/worker/ShuffleEntry.java | 2 +-
.../worker/windmill/WindmillServerStub.java | 6 +
.../windmill/client/AbstractWindmillStream.java | 6 +-
.../worker/windmill/client/WindmillStream.java | 5 +-
.../client/grpc/AppendableInputStream.java | 2 +-
.../windmill/client/grpc/GrpcCommitWorkStream.java | 4 +-
.../client/grpc/GrpcDeadlineClientInterceptor.java | 10 +-
.../client/grpc/GrpcDirectGetWorkStream.java | 4 +-
.../windmill/client/grpc/GrpcGetDataStream.java | 109 +++-
.../windmill/client/grpc/GrpcGetWorkStream.java | 4 +-
.../client/grpc/GrpcGetWorkerMetadataStream.java | 2 +-
.../windmill/client/grpc/GrpcWindmillServer.java | 41 +-
.../client/grpc/GrpcWindmillStreamFactory.java | 16 +-
.../grpc/auth/VendoredCredentialsAdapter.java | 4 +-
.../VendoredRequestMetadataCallbackAdapter.java | 4 +-
.../grpc/observers/DirectStreamObserver.java | 4 +-
.../ForwardingClientResponseObserver.java | 6 +-
.../grpc/observers/StreamObserverFactory.java | 4 +-
.../client/grpc/stubs/WindmillChannelFactory.java | 12 +-
.../client/grpc/stubs/WindmillStubFactory.java | 4 +-
.../dataflow/worker/windmill/state/StateTag.java | 2 +-
.../worker/windmill/state/WindmillBag.java | 2 +-
.../worker/windmill/state/WindmillMap.java | 2 +-
.../worker/windmill/state/WindmillMultimap.java | 2 +-
.../worker/windmill/state/WindmillOrderedList.java | 2 +-
.../worker/windmill/state/WindmillStateCache.java | 7 +-
.../worker/windmill/state/WindmillStateReader.java | 20 +-
.../worker/windmill/state/WindmillStateUtil.java | 2 +-
.../worker/windmill/state/WindmillValue.java | 2 +-
.../windmill/state/WindmillWatermarkHold.java | 2 +-
.../runners/dataflow/harness/test/TestStreams.java | 4 +-
.../runners/dataflow/worker/DataflowMatchers.java | 2 +-
.../worker/DataflowWorkUnitClientTest.java | 63 +++
.../worker/DataflowWorkerHarnessHelperTest.java | 2 +-
.../dataflow/worker/FakeWindmillServer.java | 47 +-
.../dataflow/worker/GroupingShuffleReaderTest.java | 2 +-
...ricsToPerStepNamespaceMetricsConverterTest.java | 293 +++++++++++
.../dataflow/worker/PubsubDynamicSinkTest.java | 2 +-
.../runners/dataflow/worker/PubsubReaderTest.java | 2 +-
.../runners/dataflow/worker/PubsubSinkTest.java | 2 +-
.../runners/dataflow/worker/ReaderCacheTest.java | 2 +-
.../runners/dataflow/worker/ShuffleSinkTest.java | 2 +-
.../worker/StreamingDataflowWorkerTest.java | 80 ++-
.../worker/StreamingGroupAlsoByWindowFnsTest.java | 2 +-
...reamingGroupAlsoByWindowsReshuffleDoFnTest.java | 2 +-
.../worker/StreamingModeExecutionContextTest.java | 8 +-
.../worker/StreamingSideInputDoFnRunnerTest.java | 2 +-
.../worker/StreamingSideInputFetcherTest.java | 2 +-
.../worker/StreamingStepMetricsContainerTest.java | 159 ++++++
.../runners/dataflow/worker/TestShuffleReader.java | 2 +-
.../runners/dataflow/worker/TestShuffleWriter.java | 2 +-
.../dataflow/worker/WindmillKeyedWorkItemTest.java | 2 +-
.../worker/WindmillReaderIteratorBaseTest.java | 2 +-
.../dataflow/worker/WorkerCustomSourcesTest.java | 84 ++-
.../logging/DataflowWorkerLoggingHandlerTest.java | 2 +-
.../worker/streaming/ActiveWorkStateTest.java | 52 +-
.../sideinput/SideInputStateFetcherTest.java | 16 +-
.../worker/BatchingShuffleEntryReaderTest.java | 2 +-
.../worker/GroupingShuffleEntryIteratorTest.java | 2 +-
.../util/common/worker/ShuffleEntryTest.java | 2 +-
.../grpc/GrpcGetWorkerMetadataStreamTest.java | 14 +-
.../client/grpc/GrpcWindmillServerTest.java | 207 ++++++--
.../client/grpc/StreamingEngineClientTest.java | 14 +-
.../client/grpc/WindmillStreamSenderTest.java | 6 +-
.../windmill/state/WindmillStateCacheTest.java | 2 +-
.../windmill/state/WindmillStateInternalsTest.java | 110 +++-
.../windmill/state/WindmillStateReaderTest.java | 2 +-
.../budget/EvenGetWorkBudgetDistributorTest.java | 6 +-
.../worker/windmill/src/main/proto/windmill.proto | 51 +-
runners/java-fn-execution/build.gradle | 2 +-
.../artifact/ArtifactRetrievalService.java | 8 +-
.../artifact/ArtifactStagingService.java | 10 +-
.../control/BundleCheckpointHandlers.java | 2 +-
.../fnexecution/control/FnApiControlClient.java | 6 +-
.../control/FnApiControlClientPoolService.java | 6 +-
.../control/ProcessBundleDescriptors.java | 2 +-
.../runners/fnexecution/data/GrpcDataService.java | 2 +-
.../environment/ExternalEnvironmentFactory.java | 2 +-
.../fnexecution/logging/GrpcLoggingService.java | 2 +-
.../runners/fnexecution/provisioning/JobInfo.java | 2 +-
.../provisioning/StaticGrpcProvisionService.java | 2 +-
.../fnexecution/state/GrpcStateService.java | 4 +-
.../fnexecution/state/StateRequestHandlers.java | 2 +-
.../status/BeamWorkerStatusGrpcService.java | 2 +-
.../fnexecution/status/WorkerStatusClient.java | 2 +-
.../translation/PipelineTranslatorUtils.java | 2 +-
.../runners/fnexecution/wire/ByteStringCoder.java | 4 +-
.../GrpcContextHeaderAccessorProviderTest.java | 20 +-
.../runners/fnexecution/ServerFactoryTest.java | 12 +-
.../artifact/ArtifactRetrievalServiceTest.java | 10 +-
.../artifact/ArtifactStagingServiceTest.java | 12 +-
.../control/DefaultJobBundleFactoryTest.java | 4 +-
.../control/FnApiControlClientPoolServiceTest.java | 8 +-
.../control/FnApiControlClientTest.java | 2 +-
.../fnexecution/control/RemoteExecutionTest.java | 2 +-
.../fnexecution/control/SdkHarnessClientTest.java | 2 +-
.../fnexecution/data/GrpcDataServiceTest.java | 8 +-
.../logging/GrpcLoggingServiceTest.java | 6 +-
.../StaticGrpcProvisionServiceTest.java | 10 +-
.../fnexecution/state/GrpcStateServiceTest.java | 4 +-
.../state/StateRequestHandlersTest.java | 2 +-
.../status/BeamWorkerStatusGrpcServiceTest.java | 8 +-
.../fnexecution/status/WorkerStatusClientTest.java | 2 +-
.../fnexecution/wire/ByteStringCoderTest.java | 2 +-
.../runners/fnexecution/wire/CommonCoderTest.java | 4 +-
runners/java-job-service/build.gradle | 2 +-
.../runners/jobsubmission/InMemoryJobService.java | 10 +-
.../beam/runners/jobsubmission/JobInvocation.java | 2 +-
.../beam/runners/jobsubmission/JobInvoker.java | 2 +-
.../beam/runners/jobsubmission/JobPreparation.java | 2 +-
.../jobsubmission/PortablePipelineJarCreator.java | 4 +-
.../jobsubmission/PortablePipelineJarUtils.java | 6 +-
.../jobsubmission/InMemoryJobServiceTest.java | 6 +-
.../runners/jobsubmission/JobInvocationTest.java | 2 +-
.../PortablePipelineJarCreatorTest.java | 2 +-
runners/portability/java/build.gradle | 5 +-
.../portability/JobServicePipelineResult.java | 2 +-
.../beam/runners/portability/PortableRunner.java | 4 +-
.../portability/testing/TestJobService.java | 2 +-
.../runners/portability/PortableRunnerTest.java | 8 +-
runners/samza/build.gradle | 2 +-
runners/samza/job-server/build.gradle | 2 +
.../apache/beam/runners/samza/SamzaJobInvoker.java | 2 +-
.../samza/runtime/SamzaStateRequestHandlers.java | 2 +-
.../translation/SamzaTestStreamTranslator.java | 2 +-
.../samza/translation/WindowAssignTranslator.java | 2 +-
.../runners/samza/util/PipelineJsonRenderer.java | 6 +-
.../SamzaMetricsBundleProgressHandlerTest.java | 2 +-
.../container/spark_job_server_container.gradle | 5 +-
runners/spark/job-server/spark_job_server.gradle | 4 +
runners/spark/spark_runner.gradle | 8 +-
.../apache/beam/runners/spark/SparkJobInvoker.java | 2 +-
.../beam/runners/spark/SparkPipelineRunner.java | 2 +-
runners/twister2/build.gradle | 2 +-
.../functions/AssignWindowsFunction.java | 2 +-
.../translators/functions/DoFnFunction.java | 2 +-
.../functions/GroupByWindowFunction.java | 2 +-
sdks/go.mod | 22 +-
sdks/go.sum | 76 +--
sdks/go/container/build.gradle | 5 +-
sdks/go/pkg/beam/core/core.go | 2 +-
.../pkg/beam/core/runtime/contextreg/contextreg.go | 3 +-
sdks/go/pkg/beam/core/runtime/harness/datamgr.go | 18 +-
sdks/go/pkg/beam/core/runtime/harness/init/init.go | 4 +-
sdks/go/pkg/beam/core/timers/timers.go | 7 +-
sdks/go/pkg/beam/runners/prism/internal/coders.go | 2 +-
.../pkg/beam/runners/prism/internal/engine/data.go | 16 +
.../prism/internal/engine/elementmanager.go | 414 ++++++++++++---
.../prism/internal/engine/elementmanager_test.go | 4 +-
.../runners/prism/internal/engine/engine_test.go | 14 +-
.../beam/runners/prism/internal/engine/timers.go | 174 +++++++
.../pkg/beam/runners/prism/internal/handlepardo.go | 1 -
.../beam/runners/prism/internal/jobservices/job.go | 2 +-
.../prism/internal/jobservices/management.go | 4 +-
.../runners/prism/internal/jobservices/metrics.go | 18 +-
.../pkg/beam/runners/prism/internal/preprocess.go | 2 +-
sdks/go/pkg/beam/runners/prism/internal/stage.go | 104 +++-
.../runners/prism/internal/unimplemented_test.go | 22 +
.../beam/runners/prism/internal/worker/bundle.go | 69 ++-
.../runners/prism/internal/worker/bundle_test.go | 10 +-
.../beam/runners/prism/internal/worker/worker.go | 19 +-
.../runners/prism/internal/worker/worker_test.go | 23 +-
sdks/go/test/integration/integration.go | 4 +-
sdks/java/container/agent/build.gradle | 4 +
sdks/java/container/common.gradle | 5 +-
.../container/license_scripts/dep_urls_java.yaml | 2 +-
sdks/java/core/build.gradle | 3 +-
sdks/java/core/jmh/build.gradle | 2 +-
.../jmh/util/ByteStringOutputStreamBenchmark.java | 4 +-
.../apache/beam/sdk/jmh/util/VarIntBenchmark.java | 303 +++++++++++
.../main/java/org/apache/beam/sdk/Pipeline.java | 10 +-
.../main/java/org/apache/beam/sdk/io/FileIO.java | 14 +
.../src/main/java/org/apache/beam/sdk/io/Read.java | 131 ++---
.../main/java/org/apache/beam/sdk/io/TextIO.java | 18 +
.../java/org/apache/beam/sdk/io/TextSource.java | 7 +-
.../java/org/apache/beam/sdk/io/WriteFiles.java | 12 +
.../java/org/apache/beam/sdk/io/range/ByteKey.java | 4 +-
.../beam/sdk/options/PipelineOptionsFactory.java | 19 +-
.../apache/beam/sdk/schemas/SchemaTranslation.java | 2 +-
.../schemas/logicaltypes/SchemaLogicalType.java | 2 +-
.../providers/FlattenTransformProvider.java | 100 ++++
.../providers/LoggingTransformProvider.java | 188 +++++++
.../org/apache/beam/sdk/transforms/Reshuffle.java | 6 +-
.../transforms/errorhandling/BadRecordRouter.java | 43 ++
.../beam/sdk/transforms/join/CoGbkResult.java | 3 +-
.../transforms/resourcehints/ResourceHints.java | 2 +-
.../sdk/transforms/windowing/ReshuffleTrigger.java | 3 -
.../beam/sdk/transforms/windowing/Window.java | 12 +-
.../beam/sdk/util/ByteStringOutputStream.java | 4 +-
.../java/org/apache/beam/sdk/util/CoderUtils.java | 2 +-
.../main/java/org/apache/beam/sdk/util/VarInt.java | 62 ++-
.../org/apache/beam/sdk/values/RowWithGetters.java | 21 +-
.../java/org/apache/beam/sdk/PipelineTest.java | 5 +-
.../apache/beam/sdk/SdkHarnessEnvironmentTest.java | 3 +-
.../org/apache/beam/sdk/io/TextIOReadTest.java | 113 ++--
.../org/apache/beam/sdk/io/WriteFilesTest.java | 12 +
.../beam/sdk/schemas/SchemaTranslationTest.java | 2 +-
.../beam/sdk/schemas/utils/JsonUtilsTest.java | 4 +-
.../beam/sdk/util/ByteStringOutputStreamTest.java | 2 +-
.../org/apache/beam/sdk/util/CoderUtilsTest.java | 2 +-
sdks/java/expansion-service/build.gradle | 2 +-
sdks/java/expansion-service/container/build.gradle | 5 +-
.../sdk/expansion/service/ExpansionServer.java | 4 +-
.../sdk/expansion/service/ExpansionService.java | 22 +-
.../ExpansionServiceSchemaTransformProvider.java | 5 +-
.../service/JavaClassLookupTransformProvider.java | 7 +-
.../service/WindowIntoTransformProvider.java | 127 +++++
...xpansionServiceSchemaTransformProviderTest.java | 12 +-
.../expansion/service/ExpansionServiceTest.java | 2 +-
.../beam/sdk/expansion/service/ExternalTest.java | 34 +-
.../JavaClassLookupTransformProviderTest.java | 4 +-
.../sdk/extensions/gcp/options/GcpOptions.java | 14 +-
.../sdk/extensions/gcp/util/GceMetadataUtil.java | 1 +
sdks/java/extensions/protobuf/build.gradle | 2 +
.../sdk/extensions/protobuf/ProtoByteUtils.java | 85 +++
.../extensions/protobuf/ProtoSchemaTranslator.java | 9 +-
.../extensions/protobuf/ProtoByteUtilsTest.java | 49 ++
sdks/java/extensions/python/build.gradle | 2 +-
.../extensions/python/PythonExternalTransform.java | 2 +-
.../python/ConsecutivePythonTransformsTest.java | 65 +++
.../schemaio-expansion-service/build.gradle | 2 +-
.../ExternalSchemaIOTransformRegistrar.java | 2 +-
.../extensions/sorter/BufferedExternalSorter.java | 6 +-
sdks/java/fn-execution/build.gradle | 2 +-
.../sdk/fn/channel/AddHarnessIdInterceptor.java | 8 +-
.../beam/sdk/fn/channel/ManagedChannelFactory.java | 20 +-
.../beam/sdk/fn/channel/SocketAddressFactory.java | 2 +-
.../sdk/fn/data/BeamFnDataGrpcMultiplexer.java | 4 +-
.../sdk/fn/data/BeamFnDataOutboundAggregator.java | 4 +-
.../beam/sdk/fn/data/DecodingFnDataReceiver.java | 2 +-
.../beam/sdk/fn/data/RemoteGrpcPortRead.java | 2 +-
.../beam/sdk/fn/data/RemoteGrpcPortWrite.java | 2 +-
.../org/apache/beam/sdk/fn/server/FnService.java | 6 +-
.../server/GrpcContextHeaderAccessorProvider.java | 16 +-
.../apache/beam/sdk/fn/server/GrpcFnServer.java | 2 +-
.../beam/sdk/fn/server/InProcessServerFactory.java | 8 +-
.../apache/beam/sdk/fn/server/ServerFactory.java | 20 +-
.../sdk/fn/stream/BufferingStreamObserver.java | 4 +-
.../org/apache/beam/sdk/fn/stream/DataStreams.java | 2 +-
.../beam/sdk/fn/stream/DirectStreamObserver.java | 4 +-
.../stream/ForwardingClientResponseObserver.java | 6 +-
.../sdk/fn/stream/OutboundObserverFactory.java | 4 +-
.../sdk/fn/stream/SynchronizedStreamObserver.java | 2 +-
.../org/apache/beam/sdk/fn/test/TestStreams.java | 4 +-
.../sdk/fn/windowing/EncodedBoundedWindow.java | 2 +-
.../sdk/fn/channel/ManagedChannelFactoryTest.java | 6 +-
.../sdk/fn/channel/SocketAddressFactoryTest.java | 2 +-
.../sdk/fn/data/BeamFnDataGrpcMultiplexerTest.java | 2 +-
.../beam/sdk/fn/data/RemoteGrpcPortReadTest.java | 2 +-
.../beam/sdk/fn/data/RemoteGrpcPortWriteTest.java | 2 +-
.../apache/beam/sdk/fn/stream/DataStreamsTest.java | 2 +-
.../ForwardingClientResponseObserverTest.java | 6 +-
.../sdk/fn/stream/OutboundObserverFactoryTest.java | 4 +-
.../sdk/fn/windowing/EncodedBoundedWindowTest.java | 2 +-
sdks/java/harness/build.gradle | 2 +-
sdks/java/harness/jmh/build.gradle | 2 +-
.../fn/harness/jmh/ProcessBundleBenchmark.java | 2 +-
.../control/ExecutionStateSamplerBenchmark.java | 2 +-
.../fn/harness/jmh/control/MetricsBenchmark.java | 2 +-
.../jmh/logging/BeamFnLoggingClientBenchmark.java | 6 +-
.../beam/fn/harness/BeamFnDataReadRunner.java | 2 +-
.../java/org/apache/beam/fn/harness/Caches.java | 4 +-
.../beam/fn/harness/ExternalWorkerService.java | 2 +-
.../apache/beam/fn/harness/FnApiDoFnRunner.java | 16 +-
.../java/org/apache/beam/fn/harness/FnHarness.java | 4 +-
.../fn/harness/control/BeamFnControlClient.java | 4 +-
.../fn/harness/control/BundleProgressReporter.java | 2 +-
.../fn/harness/control/ExecutionStateSampler.java | 2 +-
.../apache/beam/fn/harness/control/Metrics.java | 2 +-
.../fn/harness/control/ProcessBundleHandler.java | 4 +-
.../beam/fn/harness/data/BeamFnDataGrpcClient.java | 2 +-
.../fn/harness/logging/BeamFnLoggingClient.java | 16 +-
.../harness/state/BeamFnStateGrpcClientCache.java | 4 +-
.../beam/fn/harness/state/FnApiStateAccessor.java | 2 +-
.../fn/harness/state/FnApiTimerBundleTracker.java | 2 +-
.../beam/fn/harness/state/MultimapSideInput.java | 2 +-
.../beam/fn/harness/state/MultimapUserState.java | 2 +-
.../beam/fn/harness/state/StateBackedIterable.java | 2 +-
.../fn/harness/state/StateFetchingIterators.java | 2 +-
.../beam/fn/harness/status/BeamFnStatusClient.java | 4 +-
.../stream/HarnessStreamObserverFactories.java | 2 +-
.../beam/fn/harness/BeamFnDataReadRunnerTest.java | 2 +-
.../beam/fn/harness/BeamFnDataWriteRunnerTest.java | 2 +-
.../beam/fn/harness/ExternalWorkerServiceTest.java | 2 +-
.../beam/fn/harness/FnApiDoFnRunnerTest.java | 38 +-
.../org/apache/beam/fn/harness/FnHarnessTest.java | 8 +-
.../harness/control/BeamFnControlClientTest.java | 8 +-
.../control/BundleProgressReporterTest.java | 2 +-
.../harness/control/ExecutionStateSamplerTest.java | 2 +-
.../beam/fn/harness/control/MetricsTest.java | 2 +-
.../harness/control/ProcessBundleHandlerTest.java | 4 +-
.../fn/harness/data/BeamFnDataGrpcClientTest.java | 14 +-
.../data/PCollectionConsumerRegistryTest.java | 14 +-
.../beam/fn/harness/debug/DataSamplerTest.java | 2 +-
.../beam/fn/harness/debug/OutputSamplerTest.java | 2 +-
.../harness/logging/BeamFnLoggingClientTest.java | 32 +-
.../beam/fn/harness/state/BagUserStateTest.java | 2 +-
.../state/BeamFnStateGrpcClientCacheTest.java | 12 +-
.../fn/harness/state/FakeBeamFnStateClient.java | 2 +-
.../fn/harness/state/IterableSideInputTest.java | 2 +-
.../fn/harness/state/MultimapSideInputTest.java | 2 +-
.../fn/harness/state/MultimapUserStateTest.java | 2 +-
.../fn/harness/state/StateBackedIterableTest.java | 2 +-
.../harness/state/StateFetchingIteratorsTest.java | 2 +-
.../fn/harness/status/BeamFnStatusClientTest.java | 6 +-
.../stream/HarnessStreamObserverFactoriesTest.java | 4 +-
.../apache/beam/sdk/io/aws2/schemas/AwsTypes.java | 1 +
.../org/apache/beam/sdk/io/aws2/sqs/SqsIO.java | 4 +-
sdks/java/io/cassandra/build.gradle | 8 +
sdks/java/io/components/build.gradle | 2 +-
sdks/java/io/contextualtextio/build.gradle | 2 +-
.../contextualtextio/ContextualTextIOSource.java | 2 +-
.../csv/providers/CsvWriteTransformProvider.java | 16 +-
sdks/java/io/debezium/build.gradle | 2 +-
.../apache/beam/io/debezium/SourceRecordJson.java | 4 +-
sdks/java/io/file-schema-transform/build.gradle | 2 +-
sdks/java/io/google-cloud-platform/build.gradle | 2 +-
.../beam/sdk/io/gcp/bigquery/BatchLoads.java | 38 +-
.../beam/sdk/io/gcp/bigquery/BigQueryHelpers.java | 5 +-
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 55 +-
.../sdk/io/gcp/bigquery/BigQueryIOMetadata.java | 38 +-
.../sdk/io/gcp/bigquery/BigQueryIOTranslation.java | 28 +-
.../sdk/io/gcp/bigquery/BigQuerySinkMetrics.java | 49 +-
.../sdk/io/gcp/bigquery/CreateTableHelpers.java | 10 +-
.../gcp/bigquery/DynamicDestinationsHelpers.java | 42 +-
.../beam/sdk/io/gcp/bigquery/RetryManager.java | 2 +-
.../bigquery/StorageApiWritesShardedRecords.java | 5 +-
.../io/gcp/bigquery/UpdateSchemaDestination.java | 9 +-
.../beam/sdk/io/gcp/bigquery/WriteRename.java | 574 +++++++++++++--------
...ueryStorageWriteApiSchemaTransformProvider.java | 67 ++-
.../io/gcp/bigtable/BigtableConfigTranslator.java | 177 ++++---
.../beam/sdk/io/gcp/bigtable/BigtableIO.java | 173 ++++++-
.../beam/sdk/io/gcp/bigtable/BigtableService.java | 7 +
.../io/gcp/bigtable/BigtableServiceFactory.java | 16 +-
.../sdk/io/gcp/bigtable/BigtableServiceImpl.java | 115 ++++-
.../apache/beam/sdk/io/gcp/pubsub/PubsubIO.java | 2 +
.../beam/sdk/io/gcp/spanner/ReadSpannerSchema.java | 8 +-
.../io/gcp/spanner/SpannerTransformRegistrar.java | 2 +-
...erChangestreamsReadSchemaTransformProvider.java | 2 +-
.../changestreams/model/ValueCaptureType.java | 1 +
.../beam/sdk/io/gcp/bigquery/BigQueryIOJsonIT.java | 87 ++--
.../io/gcp/bigquery/BigQueryIOTranslationTest.java | 14 +-
.../sdk/io/gcp/bigquery/BigQueryIOWriteTest.java | 67 ++-
.../io/gcp/bigquery/BigQuerySinkMetricsTest.java | 68 ++-
...StorageWriteApiSchemaTransformProviderTest.java | 54 +-
.../gcp/bigtable/BigtableConfigTranslatorTest.java | 169 +++++-
.../beam/sdk/io/gcp/bigtable/BigtableIOTest.java | 11 +
.../beam/sdk/io/gcp/bigtable/BigtableReadIT.java | 93 +++-
.../beam/sdk/io/gcp/bigtable/BigtableWriteIT.java | 207 ++++++++
.../beam/sdk/io/gcp/healthcare/DicomIOReadIT.java | 40 +-
.../beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java | 9 +-
.../sdk/io/gcp/pubsub/PubsubIOExternalTest.java | 2 +-
.../sdk/io/gcp/spanner/SpannerIOWriteTest.java | 3 +-
.../gcp/spanner/SpannerTransformRegistrarTest.java | 2 +-
.../mapper/ChangeStreamRecordMapperTest.java | 100 ++++
sdks/java/io/hadoop-format/build.gradle | 15 +-
.../java/org/apache/beam/sdk/io/jdbc/JdbcIO.java | 27 +-
sdks/java/io/kafka/build.gradle | 1 +
.../KafkaReadSchemaTransformConfiguration.java | 23 +
.../io/kafka/KafkaReadSchemaTransformProvider.java | 157 +++---
.../kafka/KafkaWriteSchemaTransformProvider.java | 94 +++-
.../beam/sdk/io/kafka/KafkaIOExternalTest.java | 2 +-
.../KafkaReadSchemaTransformProviderTest.java | 64 +++
.../KafkaWriteSchemaTransformProviderTest.java | 22 +-
sdks/java/io/kafka/upgrade/build.gradle | 2 +-
.../sdk/io/kafka/upgrade/KafkaIOTranslation.java | 7 +-
.../io/kafka/upgrade/KafkaIOTranslationTest.java | 5 +-
.../beam/sdk/io/mongodb/MongoDbGridFSIO.java | 10 +-
.../org/apache/beam/sdk/io/mongodb/MongoDbIO.java | 2 +-
sdks/java/io/parquet/build.gradle | 2 +-
.../apache/beam/io/requestresponse/ApiIOError.java | 7 +-
.../org/apache/beam/io/requestresponse/Cache.java | 190 +++++--
.../org/apache/beam/io/requestresponse/Call.java | 377 ++++++++++----
.../beam/io/requestresponse/CallShouldBackoff.java | 2 +-
...llShouldBackoffBasedOnRejectionProbability.java | 2 +-
...ava => DefaultSerializableBackoffSupplier.java} | 24 +-
.../apache/beam/io/requestresponse/Monitoring.java | 362 +++++++++++++
.../org/apache/beam/io/requestresponse/Quota.java | 7 +-
.../beam/io/requestresponse/RedisClient.java | 4 +-
.../apache/beam/io/requestresponse/Repeater.java | 85 ++-
.../beam/io/requestresponse/RequestResponseIO.java | 552 ++++++++++++++++++--
.../org/apache/beam/io/requestresponse/Result.java | 90 ++++
.../io/requestresponse/SerializableSupplier.java} | 19 +-
.../ThrottleWithExternalResource.java | 12 +-
.../UserCodeExecutionException.java | 8 +
.../io/requestresponse/UserCodeQuotaException.java | 6 +
.../UserCodeRemoteSystemException.java | 9 +
.../requestresponse/UserCodeTimeoutException.java | 9 +
.../requestresponse/WindowedCallShouldBackoff.java | 76 +++
.../apache/beam/io/requestresponse/CacheIT.java | 16 +-
...ouldBackoffBasedOnRejectionProbabilityTest.java | 2 +-
.../apache/beam/io/requestresponse/CallTest.java | 1 -
.../EchoGRPCCallerWithSetupTeardownIT.java | 2 +-
.../beam/io/requestresponse/EchoRequestCoder.java | 1 +
...choRequestCoder.java => EchoResponseCoder.java} | 13 +-
.../beam/io/requestresponse/RedisClientIT.java | 2 +-
.../beam/io/requestresponse/RepeaterTest.java | 2 +-
.../io/requestresponse/RequestResponseIOIT.java | 95 ++++
.../io/requestresponse/RequestResponseIOTest.java | 508 ++++++++++++++++++
.../ThrottleWithExternalResourceIT.java | 6 +-
.../WindowedCallShouldBackoffTest.java | 56 ++
.../singlestore/SingleStoreDefaultRowMapper.java | 5 +-
.../testing/expansion/TestExpansionService.java | 5 +-
.../beam/sdk/tpcds/TableSchemaJSONLoader.java | 14 +-
sdks/java/transform-service/build.gradle | 2 +-
.../controller-container/build.gradle | 5 +-
.../beam/sdk/transformservice/ArtifactService.java | 6 +-
.../beam/sdk/transformservice/Controller.java | 4 +-
.../sdk/transformservice/ExpansionService.java | 4 +-
.../sdk/transformservice/ArtifactServiceTest.java | 2 +-
.../sdk/transformservice/ExpansionServiceTest.java | 2 +-
sdks/python/apache_beam/dataframe/io.py | 3 +-
.../examples/dataframe/flight_delays.py | 2 +-
.../kfp/components/preprocessing/requirements.txt | 2 +-
.../kfp/components/train/requirements.txt | 2 +-
.../io/external/xlang_bigqueryio_it_test.py | 77 ++-
sdks/python/apache_beam/io/gcp/bigquery.py | 304 ++++++-----
.../apache_beam/io/gcp/bigquery_file_loads.py | 25 +
.../apache_beam/io/gcp/bigquery_file_loads_test.py | 65 ++-
sdks/python/apache_beam/io/gcp/bigquery_test.py | 201 +++++++-
sdks/python/apache_beam/io/gcp/bigquery_tools.py | 2 +-
.../apache_beam/io/gcp/healthcare/dicomio.py | 31 +-
.../apache_beam/io/gcp/healthcare/dicomio_test.py | 39 +-
sdks/python/apache_beam/io/requestresponse.py | 413 +++++++++++++++
...nseio_it_test.py => requestresponse_it_test.py} | 37 +-
sdks/python/apache_beam/io/requestresponse_test.py | 156 ++++++
sdks/python/apache_beam/io/requestresponseio.py | 218 --------
.../apache_beam/io/requestresponseio_test.py | 88 ----
sdks/python/apache_beam/ml/transforms/base.py | 94 +++-
sdks/python/apache_beam/ml/transforms/base_test.py | 142 ++++-
.../ml/transforms/embeddings/huggingface.py | 14 +-
.../ml/transforms/embeddings/vertex_ai.py | 13 +-
sdks/python/apache_beam/ml/transforms/handlers.py | 9 +-
.../apache_beam/ml/transforms/handlers_test.py | 20 +-
sdks/python/apache_beam/portability/common_urns.py | 1 +
.../runners/dataflow/dataflow_job_service.py | 9 +-
.../runners/dataflow/dataflow_job_service_test.py | 23 +
.../runners/dataflow/dataflow_runner.py | 3 +-
.../apache_beam/runners/dataflow/internal/names.py | 2 +-
.../portability/fn_api_runner/translations.py | 2 +-
.../runners/portability/local_job_service_main.py | 4 +-
sdks/python/apache_beam/runners/render.py | 3 +-
sdks/python/apache_beam/runners/trivial_runner.py | 2 +-
.../apache_beam/runners/worker/sdk_worker.py | 11 +
.../apache_beam/testing/analyzers/perf_analysis.py | 6 +-
sdks/python/apache_beam/transforms/enrichment.py | 137 +++++
.../transforms/enrichment_handlers}/__init__.py | 0
.../transforms/enrichment_handlers/bigtable.py | 163 ++++++
.../enrichment_handlers/bigtable_it_test.py | 341 ++++++++++++
.../apache_beam/transforms/enrichment_it_test.py | 162 ++++++
.../apache_beam/transforms/enrichment_test.py} | 32 +-
..._provider.py => external_transform_provider.py} | 34 +-
...test.py => external_transform_provider_test.py} | 18 +-
.../transforms/fully_qualified_named_transform.py | 13 +-
sdks/python/apache_beam/utils/retry.py | 33 +-
sdks/python/apache_beam/version.py | 2 +-
sdks/python/apache_beam/yaml/README.md | 140 +++--
sdks/python/apache_beam/yaml/generate_yaml_docs.py | 50 +-
sdks/python/apache_beam/yaml/inline_python.md | 202 ++++++++
sdks/python/apache_beam/yaml/json_utils.py | 49 ++
sdks/python/apache_beam/yaml/main.py | 15 +-
sdks/python/apache_beam/yaml/readme_test.py | 32 +-
sdks/python/apache_beam/yaml/standard_io.yaml | 6 +
.../apache_beam/yaml/standard_providers.yaml | 27 +
sdks/python/apache_beam/yaml/yaml_mapping.md | 8 +-
sdks/python/apache_beam/yaml/yaml_mapping.py | 52 +-
sdks/python/apache_beam/yaml/yaml_provider.py | 296 +++++++++--
.../apache_beam/yaml/yaml_provider_unit_test.py | 68 +++
sdks/python/apache_beam/yaml/yaml_transform.py | 59 ++-
.../python/apache_beam/yaml/yaml_transform_test.py | 44 +-
.../apache_beam/yaml/yaml_transform_unit_test.py | 8 +-
sdks/python/container/common.gradle | 5 +-
.../container/py310/base_image_requirements.txt | 60 +--
.../container/py311/base_image_requirements.txt | 60 +--
.../container/py38/base_image_requirements.txt | 56 +-
.../container/py39/base_image_requirements.txt | 62 +--
.../expansion-service-container/build.gradle | 5 +-
sdks/python/pyproject.toml | 4 +-
sdks/python/setup.py | 9 +-
sdks/python/test-suites/dataflow/common.gradle | 2 +-
sdks/python/test-suites/tox/py38/build.gradle | 4 +
sdks/python/tox.ini | 10 +-
sdks/typescript/build.sh | 3 -
sdks/typescript/container/Dockerfile | 2 +-
sdks/typescript/container/build.gradle | 5 +-
sdks/typescript/package-lock.json | 25 +-
sdks/typescript/package.json | 4 +-
sdks/typescript/src/apache_beam/coders/coders.ts | 6 +-
.../typescript/src/apache_beam/coders/js_coders.ts | 7 +-
.../src/apache_beam/coders/required_coders.ts | 34 +-
.../typescript/src/apache_beam/coders/row_coder.ts | 24 +-
.../src/apache_beam/coders/standard_coders.ts | 7 +-
.../src/apache_beam/examples/wordcount.ts | 2 +-
.../src/apache_beam/examples/wordcount_sql.ts | 4 +-
.../src/apache_beam/examples/wordcount_textio.ts | 2 +-
.../src/apache_beam/internal/environments.ts | 12 +-
.../src/apache_beam/internal/pipeline.ts | 58 ++-
.../src/apache_beam/internal/serialize.ts | 13 +-
sdks/typescript/src/apache_beam/io/avroio.ts | 12 +-
sdks/typescript/src/apache_beam/io/bigqueryio.ts | 8 +-
sdks/typescript/src/apache_beam/io/index.ts | 3 +-
sdks/typescript/src/apache_beam/io/kafka.ts | 20 +-
sdks/typescript/src/apache_beam/io/parquetio.ts | 12 +-
sdks/typescript/src/apache_beam/io/pubsub.ts | 33 +-
sdks/typescript/src/apache_beam/io/pubsublite.ts | 12 +-
sdks/typescript/src/apache_beam/io/schemaio.ts | 10 +-
sdks/typescript/src/apache_beam/io/textio.ts | 32 +-
sdks/typescript/src/apache_beam/pvalue.ts | 57 +-
.../src/apache_beam/runners/artifacts.ts | 18 +-
.../typescript/src/apache_beam/runners/dataflow.ts | 19 +-
.../src/apache_beam/runners/direct_runner.ts | 48 +-
sdks/typescript/src/apache_beam/runners/flink.ts | 8 +-
.../apache_beam/runners/portable_runner/runner.ts | 36 +-
sdks/typescript/src/apache_beam/runners/runner.ts | 16 +-
.../src/apache_beam/runners/universal.ts | 6 +-
sdks/typescript/src/apache_beam/testing/assert.ts | 25 +-
.../apache_beam/testing/multi_pipeline_runner.ts | 22 +-
.../src/apache_beam/transforms/combiners.ts | 5 +-
.../src/apache_beam/transforms/create.ts | 2 +-
.../src/apache_beam/transforms/external.ts | 62 +--
.../src/apache_beam/transforms/flatten.ts | 4 +-
.../apache_beam/transforms/group_and_combine.ts | 59 +--
.../typescript/src/apache_beam/transforms/index.ts | 6 +-
.../src/apache_beam/transforms/internal.ts | 38 +-
.../typescript/src/apache_beam/transforms/pardo.ts | 57 +-
.../src/apache_beam/transforms/python.ts | 8 +-
sdks/typescript/src/apache_beam/transforms/sql.ts | 16 +-
.../src/apache_beam/transforms/transform.ts | 18 +-
.../typescript/src/apache_beam/transforms/utils.ts | 4 +-
.../src/apache_beam/transforms/window.ts | 22 +-
.../src/apache_beam/transforms/windowings.ts | 25 +-
.../src/apache_beam/utils/packageJson.ts | 13 +-
sdks/typescript/src/apache_beam/utils/service.ts | 51 +-
sdks/typescript/src/apache_beam/utils/utils.ts | 2 +-
sdks/typescript/src/apache_beam/values.ts | 10 +-
sdks/typescript/src/apache_beam/worker/data.ts | 6 +-
.../apache_beam/worker/external_worker_service.ts | 10 +-
sdks/typescript/src/apache_beam/worker/logging.ts | 29 +-
sdks/typescript/src/apache_beam/worker/metrics.ts | 18 +-
.../typescript/src/apache_beam/worker/operators.ts | 121 ++---
.../src/apache_beam/worker/pardo_context.ts | 26 +-
sdks/typescript/src/apache_beam/worker/state.ts | 18 +-
sdks/typescript/src/apache_beam/worker/worker.ts | 50 +-
.../src/apache_beam/worker/worker_main.ts | 8 +-
settings.gradle.kts | 2 +-
vendor/README.md | 10 +-
vendor/{grpc-1_54_0 => grpc-1_60_1}/build.gradle | 19 +-
website/Dockerfile | 2 +
website/www/site/config.toml | 2 +-
website/www/site/content/en/blog/beam-2.53.0.md | 185 +++++++
.../content/en/blog/scaling-streaming-workload.md | 2 -
website/www/site/content/en/contribute/_index.md | 2 +
.../en/documentation/dsls/sql/zetasql/overview.md | 4 +-
.../site/content/en/documentation/io/connectors.md | 2 +-
.../content/en/documentation/ml/preprocess-data.md | 75 +--
.../content/en/documentation/programming-guide.md | 4 +-
.../en/documentation/runtime/environments.md | 2 +-
.../www/site/content/en/get-started/downloads.md | 14 +-
738 files changed, 15064 insertions(+), 5246 deletions(-)
delete mode 100644 .github/trigger_files/beam_PostCommit_Go_VR_Spark.json
rename .github/trigger_files/{beam_PostCommit_Go.json =>
beam_PostCommit_Java_DataflowV2.json} (100%)
rename .github/trigger_files/{beam_PostCommit_Go_VR_Flink.json =>
beam_PostCommit_Python.json} (100%)
rename .github/trigger_files/{beam_PostCommit_Go_VR_Samza.json =>
beam_PostCommit_Python_Xlang_Gcp_Dataflow.json} (100%)
create mode 100644
.github/trigger_files/beam_PostCommit_Python_Xlang_Gcp_Direct.json
delete mode 100644 .test-infra/jenkins/job_00_seed.groovy
delete mode 100644 .test-infra/jenkins/job_Inventory.groovy
delete mode 100644
.test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_Direct.groovy
delete mode 100644
.test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_GoUsingJava_Dataflow.groovy
delete mode 100644
.test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_PythonUsingJava_Dataflow.groovy
delete mode 100644 .test-infra/jenkins/job_PostCommit_Java_DataflowV2.groovy
delete mode 100644 .test-infra/jenkins/job_PostCommit_Java_InfluxDBIO_IT.groovy
delete mode 100644
.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Flink.groovy
delete mode 100644
.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Flink_Java11.groovy
delete mode 100644 .test-infra/jenkins/job_PostCommit_Python.groovy
delete mode 100644
.test-infra/jenkins/job_PostCommit_Python_CrossLanguage_IO_Dataflow.groovy
delete mode 100644
.test-infra/jenkins/job_PostCommit_Python_Examples_Direct.groovy
delete mode 100644
.test-infra/jenkins/job_PostCommit_Python_Examples_Flink.groovy
delete mode 100644
.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Flink.groovy
delete mode 100644
.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Spark.groovy
delete mode 100644 .test-infra/jenkins/job_PostRelease_NightlySnapshot.groovy
delete mode 100644 .test-infra/jenkins/job_seed_standalone.groovy
copy
.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/{apache-beam-testing.tfvars
=> us-central1.apache-beam-testing.tfvars} (92%)
copy
.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/{apache-beam-testing.tfvars
=> us-west1.apache-beam-testing.tfvars} (97%)
rename
buildSrc/src/main/groovy/org/apache/beam/gradle/{GrpcVendoring_1_54_0.groovy =>
GrpcVendoring_1_60_1.groovy} (67%)
rename examples/notebooks/beam-ml/{mltransform =>
data_preprocessing}/compute_and_apply_vocab.ipynb (75%)
create mode 100644
examples/notebooks/beam-ml/data_preprocessing/huggingface_text_embeddings.ipynb
rename examples/notebooks/beam-ml/{mltransform =>
data_preprocessing}/scale_data.ipynb (81%)
create mode 100644
examples/notebooks/beam-ml/data_preprocessing/vertex_ai_text_embeddings.ipynb
create mode 100644
runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/UnknownCoderWrapper.java
create mode 100644
runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/EmptyUnboundedSource.java
create mode 100644
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/MetricsToPerStepNamespaceMetricsConverter.java
copy
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/{WindmillNamespacePrefix.java
=> WorkItemCancelledException.java} (55%)
create mode 100644
runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/MetricsToPerStepNamespaceMetricsConverterTest.java
create mode 100644 sdks/go/pkg/beam/runners/prism/internal/engine/timers.go
create mode 100644
sdks/java/core/jmh/src/main/java/org/apache/beam/sdk/jmh/util/VarIntBenchmark.java
create mode 100644
sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/providers/FlattenTransformProvider.java
create mode 100644
sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/providers/LoggingTransformProvider.java
create mode 100644
sdks/java/expansion-service/src/main/java/org/apache/beam/sdk/expansion/service/WindowIntoTransformProvider.java
create mode 100644
sdks/java/extensions/python/src/test/java/org/apache/beam/sdk/extensions/python/ConsecutivePythonTransformsTest.java
copy
sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/{CallShouldBackoff.java
=> DefaultSerializableBackoffSupplier.java} (61%)
create mode 100644
sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/Monitoring.java
create mode 100644
sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/Result.java
copy sdks/{typescript/src/apache_beam/io/index.ts =>
java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/SerializableSupplier.java}
(69%)
create mode 100644
sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/WindowedCallShouldBackoff.java
copy
sdks/java/io/rrio/src/test/java/org/apache/beam/io/requestresponse/{EchoRequestCoder.java
=> EchoResponseCoder.java} (74%)
create mode 100644
sdks/java/io/rrio/src/test/java/org/apache/beam/io/requestresponse/RequestResponseIOIT.java
create mode 100644
sdks/java/io/rrio/src/test/java/org/apache/beam/io/requestresponse/RequestResponseIOTest.java
create mode 100644
sdks/java/io/rrio/src/test/java/org/apache/beam/io/requestresponse/WindowedCallShouldBackoffTest.java
create mode 100644 sdks/python/apache_beam/io/requestresponse.py
rename sdks/python/apache_beam/io/{requestresponseio_it_test.py =>
requestresponse_it_test.py} (86%)
create mode 100644 sdks/python/apache_beam/io/requestresponse_test.py
delete mode 100644 sdks/python/apache_beam/io/requestresponseio.py
delete mode 100644 sdks/python/apache_beam/io/requestresponseio_test.py
create mode 100644 sdks/python/apache_beam/transforms/enrichment.py
copy {examples/notebooks/notebook_test_scripts =>
sdks/python/apache_beam/transforms/enrichment_handlers}/__init__.py (100%)
create mode 100644
sdks/python/apache_beam/transforms/enrichment_handlers/bigtable.py
create mode 100644
sdks/python/apache_beam/transforms/enrichment_handlers/bigtable_it_test.py
create mode 100644 sdks/python/apache_beam/transforms/enrichment_it_test.py
copy sdks/{typescript/build.sh =>
python/apache_beam/transforms/enrichment_test.py} (55%)
mode change 100755 => 100644
rename
sdks/python/apache_beam/transforms/{external_schematransform_provider.py =>
external_transform_provider.py} (90%)
rename
sdks/python/apache_beam/transforms/{external_schematransform_provider_test.py
=> external_transform_provider_test.py} (87%)
create mode 100644 sdks/python/apache_beam/yaml/inline_python.md
create mode 100644 sdks/python/apache_beam/yaml/yaml_provider_unit_test.py
rename
.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/apache-beam-testing.tfvars
=> sdks/typescript/src/apache_beam/utils/packageJson.ts (77%)
rename vendor/{grpc-1_54_0 => grpc-1_60_1}/build.gradle (67%)
create mode 100644 website/www/site/content/en/blog/beam-2.53.0.md
