This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from a764632db32 Document how to trigger test suites. (#30953)
add cd253fd8922 Revert setuptools upperbound since buggy releases are
yanked. (#30958)
add cfe7be6a609 [runners-flink] Use ImpulseSourceFunction for streaming
Impulse (#30903)
add f071e9e3415 Merge pull request #30905: Do not reemit data from impulse
(#30903)
add f8e4330b6ba change grafana creds to use readonly account for
beammetrics cloudsql db (#30968)
add 285b20de1ce Restore pip logs also for requirements.txt case (#30936)
add a00e947f781 Upload PerWorkerMetrics every 30 second instead of every
10 seconds (#30795)
add 4b808b0d4f5 Fix build wheels due to setuptools changed naming (#30972)
add d680bf0a57d [runners-flink] Fix watermark emission for empty splits
(#29816)
add e119cd49cf9 Merge pull request #30969: [runners-flink] Fix watermark
emission for empty splits (#29816)
add 9fa45df9b06 Move logic to build a Labeled Metric Name to a util file.
(#30796)
add 2f3893259fa [runners-flink] Finalize checkpoint marks after successful
checkpoint (#29902)
add f41f3648c3e Merge pull request #30971: #29902 finalize checkpoints
after checkpoint
add eb7ad46fcb7 remove CLOUDSDK env variable , not needed here (#30970)
add fc7b955218c Reapply "[BEAM-30531] Automatically execute unbounded
pipelines in streaming mode. (#30533)" (#30706)
add 042284afd4e Add experiment for disabling auto-streaming.
add c165f8af134 Correct release notes.
add 63ebda005d0 Merge pull request #30959 Automatically execute unbounded
pipelines in streaming mode.
add fb6bfc3ef11 Skip generating docs for yaml/integration_tests.py.
(#30986)
add a44c4f19ce3 [#30870]: support consumer polling timeout in KafkaIO
expansion service (#30915)
add e59d313c3b1 Add a link to Grafana statistics for flaky workflows
(#30934)
add db585b7b462 replace clock.milliseconds with stopwatch (#30678)
add 1a26eada0b0 [Python] Allow lambda function in bigtable handler to
build a custom row key (#30974)
add a6f3ddf898c Improvements to GetWorkTimingInfosTracker when there is
clock skew between the worker and service. (#30990)
add 995d1bf8a5a Update the upper bound for google-auth-httplib2 to
'google-auth-httplib2>=0.1.0,<0.3.0' (#30962)
add 277b6c3dbe5 Update python transform catalog (#30788)
add f6322dae132 Fix ConcurrentModification exception possible in
DataflowExecutionStateSampler (#30993)
add 2c36cc0bf49 Move dependency updates to right after branch cut (#30939)
add 1e89f834c84 Folow up of recent GHA breakages (#30996)
add 1d8fdfa83ca Remove caching of global window side input param when
invoking. (#30991)
add ae597ccbf10 Add mechanism for skipping batching if your data is
prebatched (#30961)
add ec31847d934 Skip calling Compute in RemoveSafeDeltaCounterCell if
value exists (#30913)
add a62dfa732d4 [Flink] finalize checkpoint marks in the new Flink source
implementation (#30849) (#30987)
add 6bca71070e9 Add some metrics for CoGBK profiling. (#30979)
add 6366bd4c890 Disable profiling in cythonized code by default. (#30938)
No new revisions were added by this update.
Summary of changes:
.github/trigger_files/beam_PostCommit_Python.json | 1 +
..._PostCommit_XVR_PythonUsingJavaSQL_Dataflow.yml | 2 -
...eam_PostCommit_XVR_PythonUsingJava_Dataflow.yml | 2 -
.../beam_PreCommit_Java_Kafka_IO_Direct.yml | 2 +-
.../workflows/beam_PreCommit_Yaml_Xlang_Direct.yml | 6 +-
.github/workflows/build_wheels.yml | 6 +-
.github/workflows/update_python_dependencies.yml | 12 +-
.../GA-Post-Commits_status_dashboard.json | 64 +++++++----
.../grafana/provisioning/alerting/flaky_test.yaml | 2 +-
.../metrics/kubernetes/beamgrafana-deploy.yaml | 4 +-
.test-infra/tools/flaky_test_detection.py | 26 ++++-
CHANGES.md | 5 +-
contributor-docs/release-guide.md | 20 ++--
.../Core Transforms/GroupByKey/GroupByKey/task.py | 2 +
.../flink/FlinkStreamingTransformTranslators.java | 27 ++---
.../wrappers/streaming/io/source/FlinkSource.java | 18 ++-
.../streaming/io/source/FlinkSourceReaderBase.java | 66 +++++++----
.../streaming/io/source/FlinkSourceSplit.java | 37 +++++-
.../io/source/FlinkSourceSplitEnumerator.java | 6 +-
.../io/source/bounded/FlinkBoundedSource.java | 1 +
.../source/bounded/FlinkBoundedSourceReader.java | 39 +------
.../io/source/impulse/BeamImpulseSource.java | 34 +++---
.../unbounded/FlinkUnboundedSourceReader.java | 90 ++++++++++-----
.../bounded/FlinkBoundedSourceReaderTest.java | 1 +
.../unbounded/FlinkUnboundedSourceReaderTest.java | 46 ++++++++
.../options/DataflowStreamingPipelineOptions.java | 10 ++
.../dataflow/worker/ActiveMessageMetadata.java | 7 +-
.../dataflow/worker/DataflowExecutionContext.java | 9 +-
.../worker/DataflowExecutionStateSampler.java | 14 ++-
.../MetricsToPerStepNamespaceMetricsConverter.java | 7 +-
.../worker/RemoveSafeDeltaCounterCell.java | 15 +--
.../dataflow/worker/StreamingDataflowWorker.java | 11 +-
.../runners/dataflow/worker/streaming/Work.java | 2 +-
.../harness/StreamingWorkerStatusReporter.java | 72 ++++++++++--
.../client/grpc/GetWorkTimingInfosTracker.java | 25 ++--
.../worker/DataflowExecutionContextTest.java | 7 +-
.../worker/DataflowExecutionStateSamplerTest.java | 8 +-
.../harness/StreamingWorkerStatusReporterTest.java | 10 +-
.../client/grpc/GrpcWindmillServerTest.java | 51 ++++++++-
.../beam/sdk/metrics/LabeledMetricNameUtils.java | 122 ++++++++++++++++++++
.../beam/sdk/transforms/join/CoGbkResult.java | 15 +++
.../sdk/metrics/LabeledMetricNameUtilsTest.java | 96 ++++++++++++++++
.../sdk/io/gcp/bigquery/BigQuerySinkMetrics.java | 127 ++++-----------------
.../io/gcp/bigquery/BigQuerySinkMetricsTest.java | 48 --------
.../java/org/apache/beam/sdk/io/kafka/KafkaIO.java | 25 +++-
.../beam/sdk/io/kafka/KafkaIOExternalTest.java | 5 +-
.../io/kafka/upgrade/KafkaIOTranslationTest.java | 1 +
sdks/python/apache_beam/coders/coder_impl.pxd | 1 -
.../aggregation/approximatequantiles.py} | 42 +++++--
.../aggregation/approximatequantiles_test.py} | 37 ++++--
.../transforms/aggregation/approximateunique.py} | 42 +++++--
.../aggregation/approximateunique_test.py | 54 +++++++++
.../transforms/aggregation/batchelements.py | 64 +++++++++++
.../transforms/aggregation/batchelements_test.py | 65 +++++++++++
.../snippets/transforms/aggregation/tolist.py} | 42 +++++--
.../snippets/transforms/aggregation/tolist_test.py | 60 ++++++++++
sdks/python/apache_beam/io/kafka.py | 11 +-
.../python/apache_beam/metrics/monitoring_infos.py | 1 -
sdks/python/apache_beam/ml/inference/base.py | 100 +++++++++++++++-
sdks/python/apache_beam/ml/inference/base_test.py | 8 ++
sdks/python/apache_beam/runners/common.py | 71 ++++--------
.../runners/dataflow/dataflow_runner.py | 27 +++++
.../runners/dataflow/dataflow_runner_test.py | 61 ++++++++++
.../apache_beam/runners/worker/opcounters.py | 1 -
.../apache_beam/runners/worker/operations.py | 1 -
.../runners/worker/statesampler_fast.pyx | 1 -
.../python/apache_beam/testing/fast_test_utils.pxd | 1 -
.../cy_dataflow_distribution_counter.pxd | 1 -
.../cy_dataflow_distribution_counter.pyx | 1 -
.../transforms/enrichment_handlers/bigtable.py | 26 ++++-
.../enrichment_handlers/bigtable_it_test.py | 28 +++++
.../enrichment_handlers/bigtable_test.py | 41 +++++++
sdks/python/apache_beam/utils/counters.pxd | 1 -
sdks/python/apache_beam/utils/windowed_value.py | 1 -
sdks/python/container/piputil.go | 4 +-
.../container/py310/base_image_requirements.txt | 14 +--
.../container/py311/base_image_requirements.txt | 14 +--
.../container/py38/base_image_requirements.txt | 14 +--
.../container/py39/base_image_requirements.txt | 14 +--
sdks/python/pyproject.toml | 3 +-
sdks/python/scripts/generate_pydoc.sh | 1 +
sdks/python/setup.py | 2 +-
sdks/python/tox.ini | 1 -
.../python/aggregation/approximatequantiles.md | 9 +-
.../python/aggregation/approximateunique.md | 9 +-
.../{approximatequantiles.md => batchelements.md} | 14 ++-
.../transforms/python/aggregation/groupbykey.md | 14 +--
.../{approximatequantiles.md => tolist.md} | 13 ++-
.../en/documentation/transforms/python/overview.md | 9 +-
.../partials/section-menu/en/documentation.html | 3 +
90 files changed, 1541 insertions(+), 592 deletions(-)
create mode 100644
sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/LabeledMetricNameUtils.java
create mode 100644
sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/LabeledMetricNameUtilsTest.java
copy sdks/python/apache_beam/{utils/counters.pxd =>
examples/snippets/transforms/aggregation/approximatequantiles.py} (50%)
copy sdks/python/apache_beam/{utils/counters.pxd =>
examples/snippets/transforms/aggregation/approximatequantiles_test.py} (51%)
copy sdks/python/apache_beam/{utils/counters.pxd =>
examples/snippets/transforms/aggregation/approximateunique.py} (50%)
create mode 100644
sdks/python/apache_beam/examples/snippets/transforms/aggregation/approximateunique_test.py
create mode 100644
sdks/python/apache_beam/examples/snippets/transforms/aggregation/batchelements.py
create mode 100644
sdks/python/apache_beam/examples/snippets/transforms/aggregation/batchelements_test.py
copy sdks/python/apache_beam/{utils/counters.pxd =>
examples/snippets/transforms/aggregation/tolist.py} (52%)
create mode 100644
sdks/python/apache_beam/examples/snippets/transforms/aggregation/tolist_test.py
create mode 100644
sdks/python/apache_beam/transforms/enrichment_handlers/bigtable_test.py
copy
website/www/site/content/en/documentation/transforms/python/aggregation/{approximatequantiles.md
=> batchelements.md} (57%)
copy
website/www/site/content/en/documentation/transforms/python/aggregation/{approximatequantiles.md
=> tolist.md} (65%)
