This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git.
from 30a667d Merge pull request #16727: [BEAM-11971] remove unsafe
Concurrent data structure
add 7bafabc Merge pull request #16726 from [BEAM-12164]: Parses change
streams fields as json / strings
add 6b09d10 [BEAM-13147] Avoid nullness issue during init of AwsModule
(AWS Sdk v2)
add 7e2b637 Merge pull request #16752: [BEAM-13147] Avoid nullness issue
during init of AwsModule (AWS Sdk v2)
add 98e5fc5 [BEAM-11971] Revert "Fix timer consistency in direct runner"
(#16748)
add 74d94ea [BEAM-13193] Aggregates fn api outbound data/timers of
different endpoints (#16439)
add 64581e0 [BEAM-13767] Migrate a bundle of grade tasks to use
configuration avoidance API. (#16648)
add d3b680e Merge pull request #16653 from [BEAM-12164]: Add integration
tests for spanner change streams
add d27d709 Merge pull request #16728 from [BEAM-13823] Update docs for
SnowflakeIO
add 3f75777 [BEAM-12572] Change jobs to run as cron jobs
add 93ef2f5 Merge pull request #16746 from benWize/BEAM-12572-cron-jobs
add c182e4f Merge pull request #16660 from [BEAM-13771][Playground] Send
multifile value to the frontend
add 7d76d58 Merge pull request #16646 from [BEAM-13643][Playground] Setup
running backend to verify SCIO SDK examples
add eef048b [BEAM-13015] Add state caching benchmark and move benchmarks
to their own module. (#16542)
add 87f10bb [BEAM-12976] Log projection pushdown optimizations.
add 6595861 Merge pull request #16724 from ibzib/log-opt
add 5e8c258 [BEAM-13419] Check for initialization in dataflow runner
(#16765)
add de7e9cd [BEAM-13820] Changed color of delete icon in pipeline options
dropdown, removed unused imports
add 63d9577 Merge pull request #16758 from [BEAM-13820] [Playground]
Outline delete icon in Pipeline options
add cf8af05 Merge pull request #16701 from [BEAM-13786] [Playground]
[Bugfix] Update CI/CD to verify only single-file examples
add bb15d18 Merge pull request #16754 from [BEAM-13838][Playground] Add
logs in case of empty graph for CD step
add 1ef5056 [BEAM-13293] consistent naming for expansion service address
and flag update (#16764)
add 1693483 Merge pull request #16700 from [BEAM-13790][Playground]
Change logic of receiving examples from the bucket on backend side
add 67d4b10 [BEAM-13830] update dependency for debeziumio expansion
service (#16743)
add ed41a82 [BEAM-13761] consistent namings for expansion address in
Debezium IO (#16766)
add 6bc5b66 [BEAM-13806] Shutting down SchemaIO expansion services from
Go VR script. (#16770)
add ee9bdbc [release-2.36.0] Update website/changelog for release 2.36.0
(#16627)
add 730b941 [BEAM-13848] Update numpy intersphinx link (#16767)
add 7bb7fab [release-23.6.0] Fix JIRA link for 2.36 blog (#16771)
No new revisions were added by this update.
Summary of changes:
.github/workflows/build_playground_backend.yml | 9 +
.github/workflows/build_playground_frontend.yml | 5 +-
.../job_PostCommit_Python_Examples_Dataflow.groovy | 37 +-
.../job_PostCommit_Python_Examples_Direct.groovy | 39 +-
.../job_PostCommit_Python_Examples_Flink.groovy | 39 +-
.../job_PostCommit_Python_Examples_Spark.groovy | 46 ++-
CHANGES.md | 12 +-
build.gradle.kts | 66 ++--
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 183 ++++++----
playground/api/v1/api.proto | 3 +-
playground/backend/containers/scio/Dockerfile | 2 +-
playground/backend/containers/scio/build.gradle | 2 +-
playground/backend/internal/api/v1/api.pb.go | 398 +++++++++++----------
.../internal/cloud_bucket/precompiled_objects.go | 33 +-
.../cloud_bucket/precompiled_objects_test.go | 4 +-
.../internal/utils/precompiled_objects_utils.go | 1 +
.../utils/precompiled_objects_utils_test.go | 2 +-
playground/frontend/lib/api/v1/api.pb.dart | 24 +-
playground/frontend/lib/api/v1/api.pbjson.dart | 5 +-
.../pipeline_options_dropdown_body.dart | 2 -
.../pipeline_options_form.dart | 6 +-
playground/infrastructure/api/v1/api_pb2.py | 87 ++---
playground/infrastructure/cd_helper.py | 16 +-
playground/infrastructure/ci_cd.py | 7 +-
playground/infrastructure/ci_helper.py | 8 +-
playground/infrastructure/grpc_client.py | 9 +-
playground/infrastructure/helper.py | 23 +-
.../terraform/applications/backend-router/main.tf | 15 +-
.../{backend-router => backend-scio}/main.tf | 37 +-
.../main.tf => backend-scio/variables.tf} | 73 ++--
.../graph/ProjectionPushdownOptimizer.java | 10 +
.../apache/beam/runners/core/TimerInternals.java | 17 +-
.../beam/runners/direct/DirectTimerInternals.java | 72 ++--
.../direct/ExecutorServiceParallelExecutor.java | 2 +-
.../beam/runners/direct/QuiescenceDriver.java | 78 ++--
.../direct/StatefulParDoEvaluatorFactory.java | 83 +++--
.../beam/runners/direct/WatermarkManager.java | 250 +++++++------
runners/flink/job-server/flink_job_server.gradle | 4 +-
runners/google-cloud-dataflow-java/build.gradle | 12 +-
.../worker/fn/data/BeamFnDataGrpcService.java | 10 +-
.../runners/fnexecution/data/FnDataService.java | 5 +
.../runners/fnexecution/data/GrpcDataService.java | 8 +-
.../apache/beam/runners/local/ExecutionDriver.java | 2 +-
runners/portability/java/build.gradle | 6 +-
runners/samza/job-server/build.gradle | 4 +-
runners/spark/job-server/spark_job_server.gradle | 4 +-
sdks/go/container/build.gradle | 2 +-
sdks/go/pkg/beam/io/xlang/debeziumio/debezium.go | 20 +-
sdks/go/pkg/beam/io/xlang/jdbcio/jdbc.go | 36 +-
sdks/go/pkg/beam/runners/dataflow/dataflow.go | 4 +
sdks/go/test/build.gradle | 4 +-
sdks/go/test/integration/integration.go | 11 +-
.../test/integration/io/xlang/debezium/debezium.go | 2 +-
sdks/go/test/integration/io/xlang/jdbc/jdbc.go | 4 +-
.../go/test/integration/io/xlang/jdbc/jdbc_test.go | 8 +-
sdks/go/test/run_validatesrunner_tests.sh | 3 +
.../org/apache/beam/sdk/transforms/ParDoTest.java | 10 +-
.../data/BeamFnDataBufferingOutboundObserver.java | 102 ------
.../sdk/fn/data/BeamFnDataOutboundAggregator.java | 389 ++++++++++++++++++++
.../sdk/fn/data/BeamFnDataOutboundObserver.java | 85 +++++
...amFnDataSizeBasedBufferingOutboundObserver.java | 144 --------
...amFnDataTimeBasedBufferingOutboundObserver.java | 119 ------
.../fn/data/BeamFnDataOutboundAggregatorTest.java | 386 ++++++++++++++++++++
...DataSizeBasedBufferingOutboundObserverTest.java | 210 -----------
...DataTimeBasedBufferingOutboundObserverTest.java | 178 ---------
sdks/java/harness/build.gradle | 19 -
sdks/java/harness/jmh/build.gradle | 51 +++
.../fn/harness/jmh}/ProcessBundleBenchmark.java | 222 +++++++++++-
.../jmh}/logging/BeamFnLoggingClientBenchmark.java | 4 +-
.../beam/fn/harness/jmh}/logging/package-info.java | 2 +-
.../apache/beam/fn/harness/jmh}/package-info.java | 2 +-
.../fn/harness/jmh/ProcessBundleBenchmarkTest.java | 56 +++
.../logging/BeamFnLoggingClientBenchmarkTest.java | 55 +++
.../beam/fn/harness/BeamFnDataWriteRunner.java | 123 ++-----
.../apache/beam/fn/harness/FnApiDoFnRunner.java | 42 +--
.../beam/fn/harness/PTransformRunnerFactory.java | 19 +-
.../fn/harness/control/ProcessBundleHandler.java | 113 +++---
.../beam/fn/harness/data/BeamFnDataClient.java | 20 +-
.../beam/fn/harness/data/BeamFnDataGrpcClient.java | 38 +-
.../beam/fn/harness/data/BeamFnTimerClient.java | 40 ---
.../fn/harness/data/BeamFnTimerGrpcClient.java | 48 ---
.../fn/harness/state/FnApiTimerBundleTracker.java | 5 +-
.../beam/fn/harness/BeamFnDataWriteRunnerTest.java | 187 +++-------
.../beam/fn/harness/FnApiDoFnRunnerTest.java | 78 +++-
.../PTransformRunnerFactoryTestContext.java | 76 +++-
.../harness/control/ProcessBundleHandlerTest.java | 131 +++----
.../fn/harness/data/BeamFnDataGrpcClientTest.java | 19 +-
.../fn/harness/data/FakeBeamFnTimerClient.java | 68 ----
.../apache/beam/sdk/io/aws2/options/AwsModule.java | 16 +-
sdks/java/io/expansion-service/build.gradle | 2 +
.../action/ChildPartitionsRecordAction.java | 1 -
.../action/QueryChangeStreamAction.java | 2 -
.../spanner/changestreams/dofn/InitializeDoFn.java | 7 +-
.../mapper/ChangeStreamRecordMapper.java | 34 +-
.../it/ChangeStreamTestPipelineOptions.java | 44 +++
.../changestreams/it/IntegrationTestEnv.java | 203 +++++++++++
.../changestreams/it/SpannerChangeStreamIT.java | 212 +++++++++++
.../it/SpannerChangeStreamOrderedWithinKeyIT.java | 393 ++++++++++++++++++++
.../mapper/ChangeStreamRecordMapperTest.java | 36 +-
.../changestreams/util/TestStructMapper.java | 126 +++++--
sdks/python/build.gradle | 28 +-
sdks/python/scripts/generate_pydoc.sh | 2 +-
sdks/python/test-suites/direct/common.gradle | 2 +-
sdks/python/test-suites/direct/xlang/build.gradle | 4 +-
sdks/python/test-suites/portable/common.gradle | 58 +--
settings.gradle.kts | 1 +
website/www/site/config.toml | 2 +-
website/www/site/content/en/blog/beam-2.36.0.md | 153 ++++++++
.../en/documentation/io/built-in/snowflake.md | 249 ++++++++-----
.../www/site/content/en/get-started/downloads.md | 8 +
website/www/site/data/authors.yml | 4 +
website/www/site/data/io_matrix.yaml | 3 +
website/www/site/static/.htaccess | 2 +-
113 files changed, 4066 insertions(+), 2421 deletions(-)
copy playground/terraform/applications/{backend-router =>
backend-scio}/main.tf (61%)
copy playground/terraform/applications/{backend-router/main.tf =>
backend-scio/variables.tf} (50%)
delete mode 100644
sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/data/BeamFnDataBufferingOutboundObserver.java
create mode 100644
sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/data/BeamFnDataOutboundAggregator.java
create mode 100644
sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/data/BeamFnDataOutboundObserver.java
delete mode 100644
sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/data/BeamFnDataSizeBasedBufferingOutboundObserver.java
delete mode 100644
sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/data/BeamFnDataTimeBasedBufferingOutboundObserver.java
create mode 100644
sdks/java/fn-execution/src/test/java/org/apache/beam/sdk/fn/data/BeamFnDataOutboundAggregatorTest.java
delete mode 100644
sdks/java/fn-execution/src/test/java/org/apache/beam/sdk/fn/data/BeamFnDataSizeBasedBufferingOutboundObserverTest.java
delete mode 100644
sdks/java/fn-execution/src/test/java/org/apache/beam/sdk/fn/data/BeamFnDataTimeBasedBufferingOutboundObserverTest.java
create mode 100644 sdks/java/harness/jmh/build.gradle
rename sdks/java/harness/{src/jmh/java/org/apache/beam/fn/harness =>
jmh/src/main/java/org/apache/beam/fn/harness/jmh}/ProcessBundleBenchmark.java
(57%)
rename sdks/java/harness/{src/jmh/java/org/apache/beam/fn/harness =>
jmh/src/main/java/org/apache/beam/fn/harness/jmh}/logging/BeamFnLoggingClientBenchmark.java
(97%)
rename sdks/java/harness/{src/jmh/java/org/apache/beam/fn/harness =>
jmh/src/main/java/org/apache/beam/fn/harness/jmh}/logging/package-info.java
(94%)
rename sdks/java/harness/{src/jmh/java/org/apache/beam/fn/harness =>
jmh/src/main/java/org/apache/beam/fn/harness/jmh}/package-info.java (95%)
create mode 100644
sdks/java/harness/jmh/src/test/java/org/apache/beam/fn/harness/jmh/ProcessBundleBenchmarkTest.java
create mode 100644
sdks/java/harness/jmh/src/test/java/org/apache/beam/fn/harness/jmh/logging/BeamFnLoggingClientBenchmarkTest.java
delete mode 100644
sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnTimerClient.java
delete mode 100644
sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnTimerGrpcClient.java
delete mode 100644
sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/FakeBeamFnTimerClient.java
create mode 100644
sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/it/ChangeStreamTestPipelineOptions.java
create mode 100644
sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/it/IntegrationTestEnv.java
create mode 100644
sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/it/SpannerChangeStreamIT.java
create mode 100644
sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/it/SpannerChangeStreamOrderedWithinKeyIT.java
create mode 100644 website/www/site/content/en/blog/beam-2.36.0.md
