This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from 8690ddd6bfa Update python SDK container (#32487)
add 70dd6473abd Bump github.com/docker/docker in /sdks (#32495)
add 019efe131f9 [Dataflow Streaming] Add support to read user worker
settings from backend (#32408)
add 3105260ce58 Warn when using Beam with Python 3.8 (#32484)
add ef532179d32 Fix JmsIO NPE (#32489)
add 7daeadff032 Restore release-essential references to 3.8 (#32499)
add 2cd6fb8d758 Better error message for incorrect pipeline options flags.
add e3f8c20eeea More conservative args parsing.
add e5f454b04b3 Fix mypy strictness.
add 475c98c8779 Merge pull request Better error message for incorrect
pipeline options flags.
add eb8639b013e Add throttling counter in gcsio and refactor retrying
(#32428)
add 06e2170c1ab Update python SDK container (#32501)
add aeead3f7cd3 Disable a permared suite. (#32493)
No new revisions were added by this update.
Summary of changes:
.github/build.gradle | 4 +-
.../beam_PostCommit_XVR_GoUsingJava_Dataflow.yml | 11 +-
.../workflows/beam_Publish_Beam_SDK_Snapshots.yml | 1 +
.github/workflows/build_wheels.yml | 4 +-
.github/workflows/update_python_dependencies.yml | 1 +
CHANGES.md | 2 +
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 3 +-
gradle.properties | 2 +-
local-env-setup.sh | 4 +-
release/src/main/Dockerfile | 3 +-
.../python-release/python_release_automation.sh | 2 +-
.../runners/dataflow/worker/OperationalLimits.java | 39 +--
.../dataflow/worker/StreamingDataflowWorker.java | 75 ++----
.../worker/StreamingModeExecutionContext.java | 12 +-
.../worker/streaming/ComputationWorkExecutor.java | 5 +-
.../worker/streaming/config/ComputationConfig.java | 3 +-
.../streaming/config/FixedGlobalConfigHandle.java | 31 ++-
...StreamingApplianceComputationConfigFetcher.java | 10 +-
.../StreamingEngineComputationConfigFetcher.java | 69 +++--
...elineConfig.java => StreamingGlobalConfig.java} | 41 ++-
.../config/StreamingGlobalConfigHandle.java | 23 +-
.../config/StreamingGlobalConfigHandleImpl.java | 113 ++++++++
.../harness/StreamingWorkerStatusPages.java | 22 +-
.../windmill/client/grpc/GrpcDispatcherClient.java | 10 +
.../processing/ComputationWorkExecutorFactory.java | 7 +-
.../work/processing/StreamingWorkScheduler.java | 25 +-
.../worker/StreamingDataflowWorkerTest.java | 45 +++-
.../worker/StreamingModeExecutionContextTest.java | 8 +-
.../dataflow/worker/WorkerCustomSourcesTest.java | 11 +-
.../config/FixedGlobalConfigHandleTest.java | 84 ++++++
...amingApplianceComputationConfigFetcherTest.java | 4 +-
...treamingEngineComputationConfigFetcherTest.java | 102 ++++---
.../StreamingGlobalConfigHandleImplTest.java | 293 +++++++++++++++++++++
.../worker/windmill/src/main/proto/windmill.proto | 9 +
sdks/go.mod | 2 +-
sdks/go.sum | 4 +-
.../java/org/apache/beam/sdk/io/jms/JmsIO.java | 35 ++-
sdks/python/apache_beam/__init__.py | 6 +
sdks/python/apache_beam/io/gcp/gcsio.py | 73 +++--
.../apache_beam/io/gcp/gcsio_integration_test.py | 39 ++-
sdks/python/apache_beam/io/gcp/gcsio_retry.py | 71 +++++
sdks/python/apache_beam/io/gcp/gcsio_retry_test.py | 84 ++++++
sdks/python/apache_beam/io/gcp/gcsio_test.py | 16 +-
.../python/apache_beam/options/pipeline_options.py | 26 +-
.../apache_beam/runners/dataflow/internal/names.py | 2 +-
sdks/python/setup.py | 5 +
46 files changed, 1149 insertions(+), 292 deletions(-)
copy
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/InMemoryMetaTableProvider.java
=>
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/config/FixedGlobalConfigHandle.java
(55%)
rename
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/config/{StreamingEnginePipelineConfig.java
=> StreamingGlobalConfig.java} (56%)
copy
sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/payloads/PayloadSerializerProvider.java
=>
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/config/StreamingGlobalConfigHandle.java
(59%)
create mode 100644
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/config/StreamingGlobalConfigHandleImpl.java
create mode 100644
runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/streaming/config/FixedGlobalConfigHandleTest.java
create mode 100644
runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/streaming/config/StreamingGlobalConfigHandleImplTest.java
create mode 100644 sdks/python/apache_beam/io/gcp/gcsio_retry.py
create mode 100644 sdks/python/apache_beam/io/gcp/gcsio_retry_test.py
