This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from 5d13894975b Revert "Update Build Wheels to only build once on RCs"
(#32014)
add 25a4ffe8e09 Bump setuptools from 68.2.2 to 70.0.0 in
/.test-infra/mock-apis (#31893)
add ee3d57fb13c Bump com.gradle.develocity from 3.17.5 to 3.17.6 (#31948)
add 17ef888a783 Add StringSet metrics to Python SDK (#31969)
add 89d5e2f2961 Validate commits in StreamingDataflowWorker (#31822)
add 98f8b869327 beam-sql-udf-doc-mistake (#32019)
add 570f2f89647 Fix/Refactor GetData interfaces for Direct Path
integration (#31784)
add c624e02b852 [CsvIO] update error and result handling. (#32023)
add 121ac713fa0 [#31991][prism] Allow Empty Composites (#32024)
add 88a01027f4c [#32003][prism] Support empty transform input sets, such
as for flattens. (#32029)
add e7847c9448c [#32004] Ensure all pcollection coders are length prefixed
if necessary. (#32012)
No new revisions were added by this update.
Summary of changes:
.test-infra/mock-apis/poetry.lock | 13 +-
.../worker/MetricTrackingWindmillServerStub.java | 355 ---------------------
.../runners/dataflow/worker/OperationalLimits.java | 64 ++++
...Exception.java => OutputTooLargeException.java} | 19 +-
.../dataflow/worker/StreamingDataflowWorker.java | 131 +++++---
.../worker/StreamingModeExecutionContext.java | 18 ++
.../beam/runners/dataflow/worker/WindmillSink.java | 25 ++
.../worker/WorkItemCancelledException.java | 4 +
.../dataflow/worker/streaming/ActiveWorkState.java | 61 ++--
.../worker/streaming/ComputationState.java | 13 +-
.../worker/streaming/ComputationWorkExecutor.java | 7 +-
.../dataflow/worker/streaming/ExecutableWork.java | 2 +-
.../{ExecutableWork.java => RefreshableWork.java} | 34 +-
.../runners/dataflow/worker/streaming/Work.java | 72 +++--
.../runners/dataflow/worker/streaming/WorkId.java | 4 +-
.../StreamingEngineComputationConfigFetcher.java | 16 +-
.../config/StreamingEnginePipelineConfig.java | 10 +
.../streaming/sideinput/SideInputStateFetcher.java | 114 +++----
.../sideinput/SideInputStateFetcherFactory.java | 46 +++
.../worker/windmill/ApplianceWindmillClient.java | 39 +++
.../windmill/StreamingEngineWindmillClient.java | 54 ++++
.../worker/windmill/WindmillConnection.java | 13 +-
.../worker/windmill/WindmillServerBase.java | 5 -
.../worker/windmill/WindmillServerStub.java | 58 +---
.../windmill/client/AbstractWindmillStream.java | 56 +++-
.../worker/windmill/client/WindmillStream.java | 27 +-
.../worker/windmill/client/WindmillStreamPool.java | 4 +-
.../commits/StreamingEngineWorkCommitter.java | 113 ++++---
.../client/getdata/ApplianceGetDataClient.java | 220 +++++++++++++
.../windmill/client/getdata/GetDataClient.java | 57 ++++
.../client/getdata/StreamGetDataClient.java | 101 ++++++
.../client/getdata/StreamPoolGetDataClient.java | 80 +++++
.../getdata/ThrottlingGetDataMetricTracker.java | 108 +++++++
.../windmill/client/grpc/GrpcCommitWorkStream.java | 7 +-
.../client/grpc/GrpcDirectGetWorkStream.java | 32 +-
.../windmill/client/grpc/GrpcGetDataStream.java | 22 +-
.../windmill/client/grpc/GrpcGetWorkStream.java | 11 +-
.../client/grpc/GrpcGetWorkerMetadataStream.java | 4 +-
.../windmill/client/grpc/GrpcWindmillServer.java | 5 -
.../client/grpc/GrpcWindmillStreamFactory.java | 18 +-
.../client/grpc/StreamingEngineClient.java | 28 +-
.../windmill/client/grpc/WindmillStreamSender.java | 28 +-
.../StreamObserverCancelledException.java} | 26 +-
.../work/processing/StreamingWorkScheduler.java | 38 ++-
.../windmill/work/refresh/ActiveWorkRefresher.java | 101 +++++-
.../work/refresh/ActiveWorkRefreshers.java | 50 ---
.../work/refresh/ApplianceHeartbeatSender.java | 62 ++++
.../refresh/DispatchedActiveWorkRefresher.java | 68 ----
.../work/refresh/FixedStreamHeartbeatSender.java | 93 ++++++
.../work/refresh/HeartbeatSender.java} | 33 +-
.../worker/windmill/work/refresh/Heartbeats.java | 70 ++++
.../work/refresh/StreamPoolHeartbeatSender.java | 48 +++
.../dataflow/worker/FakeWindmillServer.java | 84 ++---
.../worker/StreamingDataflowWorkerTest.java | 118 +++++--
.../worker/StreamingModeExecutionContextTest.java | 14 +-
.../dataflow/worker/WorkerCustomSourcesTest.java | 13 +-
.../worker/streaming/ActiveWorkStateTest.java | 77 +----
.../streaming/ComputationStateCacheTest.java | 7 +-
.../sideinput/SideInputStateFetcherTest.java | 93 +++---
.../worker/util/BoundedQueueExecutorTest.java | 26 +-
.../windmill/client/WindmillStreamPoolTest.java | 14 +-
.../StreamingApplianceWorkCommitterTest.java | 13 +-
.../commits/StreamingEngineWorkCommitterTest.java | 54 +++-
.../client/getdata/FakeGetDataClient.java} | 35 +-
.../ThrottlingGetDataMetricTrackerTest.java | 277 ++++++++++++++++
.../grpc/GrpcGetWorkerMetadataStreamTest.java | 2 +-
.../client/grpc/GrpcWindmillServerTest.java | 27 +-
.../client/grpc/StreamingEngineClientTest.java | 21 +-
.../client/grpc/WindmillStreamSenderTest.java | 51 +--
.../windmill/state/WindmillStateReaderTest.java | 4 +-
.../budget/EvenGetWorkBudgetDistributorTest.java | 5 +-
.../failures/WorkFailureProcessorTest.java | 8 +-
...esherTest.java => ActiveWorkRefresherTest.java} | 59 ++--
sdks/go/pkg/beam/core/runtime/exec/hash.go | 7 +
.../prism/internal/engine/elementmanager.go | 37 ++-
.../prism/internal/jobservices/management.go | 15 +-
sdks/go/pkg/beam/runners/prism/internal/stage.go | 33 +-
.../java/org/apache/beam/sdk/metrics/Metrics.java | 10 +-
.../beam/sdk/io/csv/CsvIOParseConfiguration.java | 25 --
.../apache/beam/sdk/io/csv/CsvIOParseError.java | 24 ++
.../org/apache/beam/sdk/io/csv/CsvIOParseKV.java | 15 +-
.../apache/beam/sdk/io/csv/CsvIOParseResult.java | 100 ++++++
.../beam/sdk/io/csv/CsvIORecordToObjects.java | 51 ++-
.../beam/sdk/io/csv/CsvIOStringToCsvRecord.java | 45 ++-
.../beam/sdk/io/csv/CsvIORecordToObjectsTest.java | 54 +++-
.../sdk/io/csv/CsvIOStringToCsvRecordTest.java | 109 +++++--
sdks/python/apache_beam/metrics/cells.pxd | 6 +
sdks/python/apache_beam/metrics/cells.py | 75 +++++
sdks/python/apache_beam/metrics/cells_test.py | 24 ++
sdks/python/apache_beam/metrics/execution.py | 20 +-
sdks/python/apache_beam/metrics/execution_test.py | 9 +
sdks/python/apache_beam/metrics/metric.py | 31 +-
sdks/python/apache_beam/metrics/metricbase.py | 16 +-
.../python/apache_beam/metrics/monitoring_infos.py | 53 ++-
.../apache_beam/metrics/monitoring_infos_test.py | 25 ++
.../runners/dataflow/dataflow_metrics.py | 15 +-
.../apache_beam/runners/direct/direct_metrics.py | 15 +-
.../runners/direct/direct_runner_test.py | 9 +
.../runners/portability/fn_api_runner/fn_runner.py | 11 +-
.../portability/fn_api_runner/fn_runner_test.py | 9 +-
.../runners/portability/portable_metrics.py | 13 +-
.../runners/portability/portable_runner.py | 5 +-
settings.gradle.kts | 2 +-
.../dsls/sql/extensions/user-defined-functions.md | 2 +-
104 files changed, 3174 insertions(+), 1380 deletions(-)
delete mode 100644
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/MetricTrackingWindmillServerStub.java
create mode 100644
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/OperationalLimits.java
copy
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/{WorkItemCancelledException.java
=> OutputTooLargeException.java} (63%)
copy
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/{ExecutableWork.java
=> RefreshableWork.java} (58%)
create mode 100644
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/sideinput/SideInputStateFetcherFactory.java
create mode 100644
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/ApplianceWindmillClient.java
create mode 100644
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/StreamingEngineWindmillClient.java
create mode 100644
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/client/getdata/ApplianceGetDataClient.java
create mode 100644
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/client/getdata/GetDataClient.java
create mode 100644
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/client/getdata/StreamGetDataClient.java
create mode 100644
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/client/getdata/StreamPoolGetDataClient.java
create mode 100644
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/client/getdata/ThrottlingGetDataMetricTracker.java
copy
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/{WorkItemCancelledException.java
=> windmill/client/grpc/observers/StreamObserverCancelledException.java} (53%)
delete mode 100644
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/work/refresh/ActiveWorkRefreshers.java
create mode 100644
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/work/refresh/ApplianceHeartbeatSender.java
delete mode 100644
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/work/refresh/DispatchedActiveWorkRefresher.java
create mode 100644
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/work/refresh/FixedStreamHeartbeatSender.java
copy
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/{WorkItemCancelledException.java
=> windmill/work/refresh/HeartbeatSender.java} (52%)
create mode 100644
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/work/refresh/Heartbeats.java
create mode 100644
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/work/refresh/StreamPoolHeartbeatSender.java
copy
runners/google-cloud-dataflow-java/worker/src/{main/java/org/apache/beam/runners/dataflow/worker/streaming/ExecutableWork.java
=>
test/java/org/apache/beam/runners/dataflow/worker/windmill/client/getdata/FakeGetDataClient.java}
(55%)
create mode 100644
runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/client/getdata/ThrottlingGetDataMetricTrackerTest.java
rename
runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/work/refresh/{DispatchedActiveWorkRefresherTest.java
=> ActiveWorkRefresherTest.java} (84%)
create mode 100644
sdks/java/io/csv/src/main/java/org/apache/beam/sdk/io/csv/CsvIOParseResult.java
