This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git.
from f1ff75e Merge pull request #16146: [BEAM-13352] don't store
unserializable DatasetService as a member variable
add dc5bd89 Merge pull request #16127 from [BEAM-13366] [Playground] Add
support Pipeline Options for Backend
add 7b71039 [BEAM-11758] Final cleanup for Beam Basics doc content
add 82c7c12 Merge pull request #16157: [BEAM-11758] Final cleanup for
Beam Basics doc content
add ac4474f [BEAM-13092] Adding dummy external transform translators for
Dataflow runner
add 4e3c00d Merge pull request #15766: [BEAM-13092] Adding dummy external
transform translators for Dataflow…
add d6caea5 Merge pull request #16167 from [BEAM-13409][Playground]
[Bugfix] Change title to find beam-playground examples
add f0a784a fix playground frontend licences
add 5ee3e9d Merge pull request #16169 from [BEAM-13412] [Bugfix]
[Playground] fix playground frontend licences
add 13a8746 Merge pull request #16136 from [BEAM-13365] [Playground] Add
Pipelines options text field to the frontend
add 918f345 [BEAM-13244] Support STS Assume role credentials provider for
AWS SDK v2 (#16116)
add b52762b [BEAM-11936] Fix errorprone UnusedVariable in
core,examples,harness.. (#16037)
add 4147070 [BEAM-11936] Remove suppressUnusedVariable flag (#16171)
add e52bdf2 [BEAM-13090] Adding SDK harness container overrides option to
Java SDK (#15764)
add 06a5e67 [BEAM-11936] Fix errorprone warnings (#15890)
add 774008d [BEAM-13015] Start integrating a process wide cache. (#16130)
No new revisions were added by this update.
Summary of changes:
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 15 -
examples/java/build.gradle | 1 -
.../examples/complete/game/StatefulTeamScore.java | 3 +-
.../beam/examples/cookbook/JoinExamples.java | 3 +-
.../apache/beam/examples/snippets/Snippets.java | 11 +-
.../examples/subprocess/ExampleEchoPipeline.java | 1 -
.../subprocess/kernel/SubProcessKernel.java | 2 -
.../examples/subprocess/utils/ExecutableFile.java | 4 -
.../examples/complete/TrafficMaxLaneFlowIT.java | 4 +-
.../beam/examples/complete/TrafficRoutesIT.java | 3 -
.../datatokenization/DataTokenizationTest.java | 2 -
.../subprocess/ExampleEchoPipelineTest.java | 2 -
examples/java/twitter/build.gradle | 1 -
examples/kotlin/build.gradle | 2 +-
playground/api/v1/api.proto | 2 +
playground/backend/cmd/server/controller.go | 2 +-
playground/backend/internal/api/v1/api.pb.go | 369 ++--
.../internal/code_processing/code_processing.go | 4 +-
.../code_processing/code_processing_test.go | 68 +-
playground/backend/internal/executors/executor.go | 17 +-
.../backend/internal/executors/executor_builder.go | 8 +
.../backend/internal/executors/executor_test.go | 10 +-
.../internal/setup_tools/builder/setup_builder.go | 7 +-
.../setup_tools/builder/setup_builder_test.go | 18 +-
.../setup_tools/life_cycle/life_cycle_setuper.go | 5 +-
.../life_cycle/life_cycle_setuper_test.go | 1 -
.../backend/internal/utils/common.go | 9 +-
.../utils/{system_utils_test.go => common_test.go} | 15 +-
.../backend/internal/utils/preparators_utils.go | 7 +
...tem_utils_test.go => preparators_utils_test.go} | 24 +-
.../backend/internal/utils/system_utils_test.go | 4 +-
playground/frontend/lib/api/v1/api.pb.dart | 227 +--
playground/frontend/lib/api/v1/api.pbgrpc.dart | 120 +-
playground/frontend/lib/api/v1/api.pbjson.dart | 78 +-
playground/frontend/lib/constants/sizes.dart | 2 +
.../components/pipeline_options_text_field.dart | 81 +
.../modules/editor/parsers/run_options_parser.dart | 57 +
.../code_client/grpc_code_client.dart | 4 +-
.../code_repository/run_code_request.dart | 7 +-
.../lib/modules/examples/models/example_model.dart | 2 +
.../example_client/grpc_example_client.dart | 22 +-
.../components/editor_textarea_wrapper.dart | 53 +-
.../pages/playground/states/playground_state.dart | 37 +-
.../editor/parsers/run_options_parser_test.dart | 60 +
.../code_repository/code_repository_test.dart | 1 +
playground/infrastructure/api/v1/api_pb2.py | 1948 +++++++++++---------
playground/infrastructure/api/v1/api_pb2_grpc.py | 437 +++--
playground/infrastructure/config.py | 4 +-
playground/infrastructure/grpc_client.py | 7 +-
playground/infrastructure/helper.py | 5 +-
playground/infrastructure/test_grpc_client.py | 2 +-
.../core/construction/PTransformMatchersTest.java | 21 -
.../beam/runners/direct/NanosOffsetClock.java | 6 +-
.../runners/direct/TransformEvaluatorRegistry.java | 16 -
.../runners/direct/UnboundedReadDeduplicator.java | 3 +-
.../runners/direct/CloningBundleFactoryTest.java | 67 -
.../flink/FlinkStreamingPipelineTranslator.java | 4 +-
.../flink/FlinkStreamingTransformTranslators.java | 17 -
.../wrappers/streaming/DoFnOperator.java | 2 +-
.../state/FlinkBroadcastStateInternals.java | 131 --
.../streaming/ExecutableStageDoFnOperatorTest.java | 1 +
.../beam/runners/dataflow/DataflowRunner.java | 168 +-
.../options/DataflowPipelineDebugOptions.java | 9 +
.../runners/dataflow/DataflowPipelineJobTest.java | 25 -
.../beam/runners/dataflow/DataflowRunnerTest.java | 42 +
.../beam/runners/dataflow/worker/ReaderCache.java | 3 +-
.../beam/runners/dataflow/worker/StateFetcher.java | 4 +-
.../dataflow/worker/StreamingDataflowWorker.java | 2 +-
.../fn/data/RemoteGrpcPortWriteOperation.java | 6 +-
.../common/worker/CachingShuffleBatchReader.java | 4 +-
.../control/DefaultJobBundleFactory.java | 16 +-
.../environment/EmbeddedEnvironmentFactory.java | 4 +-
.../fnexecution/control/RemoteExecutionTest.java | 6 +-
.../beam/runners/spark/io/MicrobatchSource.java | 2 +-
.../translation/utils/SideInputStorage.java | 4 +-
.../runners/spark/util/GlobalWatermarkHolder.java | 4 +-
.../beam/runners/spark/util/SideInputStorage.java | 4 +-
sdks/java/build-tools/build.gradle | 2 +-
.../apache/beam/codestyle/CodeStyleTestClass.java | 2 -
sdks/java/core/build.gradle | 1 -
.../apache/beam/sdk/coders/RowCoderGenerator.java | 4 -
.../src/main/java/org/apache/beam/sdk/io/Read.java | 3 +-
.../beam/sdk/io/ReadAllViaFileBasedSource.java | 3 -
.../beam/sdk/metrics/MetricQueryResults.java | 1 -
.../beam/sdk/options/PipelineOptionsFactory.java | 5 +-
.../apache/beam/sdk/options/SdkHarnessOptions.java | 72 +
.../java/org/apache/beam/sdk/schemas/Schema.java | 4 -
.../org/apache/beam/sdk/schemas/SchemaCoder.java | 20 -
.../apache/beam/sdk/schemas/transforms/Filter.java | 3 +-
.../sdk/schemas/transforms/SchemaAggregateFn.java | 2 -
.../apache/beam/sdk/schemas/transforms/Select.java | 6 -
.../beam/sdk/schemas/utils/AutoValueUtils.java | 2 -
.../org/apache/beam/sdk/state/StateKeySpec.java | 14 +-
.../apache/beam/sdk/transforms/Deduplicate.java | 2 +
.../beam/sdk/transforms/DoFnSchemaInformation.java | 17 +-
.../beam/sdk/transforms/GroupIntoBatches.java | 10 +-
.../java/org/apache/beam/sdk/transforms/ParDo.java | 4 +-
.../java/org/apache/beam/sdk/transforms/Watch.java | 4 -
.../reflect/ByteBuddyDoFnInvokerFactory.java | 3 -
.../reflect/ByteBuddyOnTimerInvokerFactory.java | 6 -
.../beam/sdk/transforms/reflect/DoFnInvokers.java | 6 +-
.../sdk/transforms/reflect/DoFnSignatures.java | 15 -
.../apache/beam/sdk/values/PCollectionViews.java | 59 -
.../main/java/org/apache/beam/sdk/values/Row.java | 1 -
.../apache/beam/sdk/coders/CoderRegistryTest.java | 4 -
.../sdk/coders/PCollectionCustomCoderTest.java | 3 -
.../org/apache/beam/sdk/io/TextIOWriteTest.java | 17 +-
.../beam/sdk/io/TextRowCountEstimatorTest.java | 5 +-
.../apache/beam/sdk/io/range/ByteKeyRangeTest.java | 4 +-
.../apache/beam/sdk/schemas/AvroSchemaTest.java | 3 -
.../sdk/schemas/FieldAccessDescriptorTest.java | 4 +-
.../beam/sdk/schemas/JavaBeanSchemaTest.java | 2 +-
.../apache/beam/sdk/schemas/SchemaOptionsTest.java | 2 +-
.../beam/sdk/schemas/transforms/CoGroupTest.java | 48 +-
.../beam/sdk/schemas/transforms/GroupTest.java | 10 +-
.../beam/sdk/schemas/transforms/SelectTest.java | 13 +-
.../beam/sdk/schemas/utils/SelectHelpersTest.java | 6 +-
.../apache/beam/sdk/testing/ExpectedLogsTest.java | 10 +-
.../beam/sdk/testing/SystemNanoTimeSleeper.java | 4 +-
.../apache/beam/sdk/testing/TestStreamTest.java | 3 +
.../apache/beam/sdk/transforms/GroupByKeyTest.java | 3 +-
.../org/apache/beam/sdk/transforms/LatestTest.java | 3 -
.../beam/sdk/transforms/ParDoLifecycleTest.java | 1 +
.../beam/sdk/transforms/ParDoSchemaTest.java | 4 +
.../org/apache/beam/sdk/transforms/ParDoTest.java | 47 +-
.../beam/sdk/transforms/PerKeyOrderingTest.java | 9 +-
.../beam/sdk/transforms/WithFailuresTest.java | 3 -
.../sdk/transforms/display/DisplayDataTest.java | 6 +-
.../sdk/transforms/reflect/DoFnInvokersTest.java | 5 +-
.../reflect/DoFnSignaturesSplittableDoFnTest.java | 16 +-
.../sdk/transforms/reflect/DoFnSignaturesTest.java | 90 +-
.../transforms/reflect/OnTimerInvokersTest.java | 2 +
.../reflect/testhelper/DoFnInvokersTestHelper.java | 7 +
.../GrowableOffsetRangeTrackerTest.java | 4 +-
.../splittabledofn/OffsetRangeTrackerTest.java | 2 +-
.../beam/sdk/util/MutationDetectorsTest.java | 2 +-
.../java/org/apache/beam/sdk/values/RowTest.java | 4 +-
sdks/java/expansion-service/app/build.gradle | 1 -
sdks/java/expansion-service/build.gradle | 1 -
.../JavaClassLookupTransformProviderTest.java | 2 -
.../core/translate/TimestampExtractTransform.java | 8 -
sdks/java/extensions/sql/hcatalog/build.gradle | 1 -
.../sql/meta/provider/kafka/BeamKafkaTable.java | 3 +-
.../extensions/sql/udf-test-provider/build.gradle | 1 -
sdks/java/fn-execution/build.gradle | 1 -
.../org/apache/beam/sdk/fn/CancellableQueue.java | 4 +-
sdks/java/harness/build.gradle | 12 +-
.../beam/fn/harness/ProcessBundleBenchmark.java | 7 +-
.../beam/fn/harness/BeamFnDataWriteRunner.java | 4 -
.../java/org/apache/beam/fn/harness/Cache.java | 65 +
.../java/org/apache/beam/fn/harness/Caches.java | 259 +++
.../apache/beam/fn/harness/FnApiDoFnRunner.java | 10 +-
.../java/org/apache/beam/fn/harness/FnHarness.java | 48 +-
.../fn/harness/control/FinalizeBundleHandler.java | 17 +-
.../fn/harness/control/ProcessBundleHandler.java | 9 +-
.../harness/data/PCollectionConsumerRegistry.java | 2 -
.../beam/fn/harness/status/MemoryMonitor.java | 1 +
.../org/apache/beam/fn/harness/CachesTest.java | 152 ++
.../beam/fn/harness/FnApiDoFnRunnerTest.java | 9 +-
.../harness/control/ProcessBundleHandlerTest.java | 41 +-
.../state/BeamFnStateGrpcClientCacheTest.java | 8 -
.../state/CachingBeamFnStateClientTest.java | 8 +-
.../harness/state/StateFetchingIteratorsTest.java | 2 +-
.../fn/harness/status/BeamFnStatusClientTest.java | 11 +-
sdks/java/io/amazon-web-services2/build.gradle | 2 +-
.../apache/beam/sdk/io/aws2/options/AwsModule.java | 62 +-
.../beam/sdk/io/aws2/options/AwsModuleTest.java | 179 +-
.../bigquery/StorageApiWriteUnshardedRecords.java | 3 +-
.../bigquery/StorageApiWritesShardedRecords.java | 3 +-
.../internal/LimitingTopicBacklogReader.java | 6 +-
.../beam/sdk/io/gcp/spanner/SpannerAccessor.java | 24 -
.../sdk/io/hadoop/format/TestRowDBWritable.java | 10 -
.../beam/sdk/io/kafka/KafkaExactlyOnceSink.java | 3 +-
.../beam/sdk/io/kafka/KafkaUnboundedReader.java | 4 +-
.../beam/sdk/io/kafka/ReadFromKafkaDoFn.java | 5 +-
.../org/apache/beam/sdk/io/xml/XmlSourceTest.java | 4 +-
sdks/java/javadoc/build.gradle | 2 +-
sdks/java/maven-archetypes/examples/build.gradle | 2 +-
.../maven-archetypes/gcp-bom-examples/build.gradle | 2 +-
sdks/java/maven-archetypes/starter/build.gradle | 2 +-
sdks/java/testing/expansion-service/build.gradle | 1 -
sdks/java/testing/jpms-tests/build.gradle | 1 -
sdks/java/testing/kafka-service/build.gradle | 2 +-
sdks/java/testing/load-tests/build.gradle | 1 -
.../beam/sdk/loadtests/SideInputLoadTest.java | 3 +
sdks/java/testing/nexmark/build.gradle | 1 -
.../java/org/apache/beam/sdk/nexmark/Main.java | 7 +-
.../apache/beam/sdk/nexmark/NexmarkLauncher.java | 3 -
.../org/apache/beam/sdk/nexmark/NexmarkUtils.java | 3 +-
.../beam/sdk/nexmark/model/sql/SelectEvent.java | 1 -
.../beam/sdk/nexmark/queries/NexmarkQuery.java | 7 -
.../apache/beam/sdk/nexmark/queries/Query10.java | 1 +
.../apache/beam/sdk/nexmark/queries/Query3.java | 7 +-
.../queries/sql/SqlBoundedSideInputJoin.java | 3 -
.../beam/sdk/nexmark/queries/sql/SqlQuery0.java | 6 +-
.../beam/sdk/nexmark/queries/sql/SqlQuery2.java | 2 +-
sdks/java/testing/test-utils/build.gradle | 1 -
sdks/java/testing/tpcds/build.gradle | 1 -
sdks/java/testing/watermarks/build.gradle | 1 -
sdks/python/test-suites/direct/xlang/build.gradle | 2 +-
.../www/site/content/en/documentation/basics.md | 116 +-
.../www/site/content/en/documentation/glossary.md | 35 +-
.../content/en/documentation/programming-guide.md | 14 +-
203 files changed, 3329 insertions(+), 2856 deletions(-)
copy sdks/go/pkg/beam/core/runtime/graphx/v1/gen.go =>
playground/backend/internal/utils/common.go (84%)
copy playground/backend/internal/utils/{system_utils_test.go =>
common_test.go} (71%)
copy playground/backend/internal/utils/{system_utils_test.go =>
preparators_utils_test.go} (62%)
create mode 100644
playground/frontend/lib/modules/editor/components/pipeline_options_text_field.dart
create mode 100644
playground/frontend/lib/modules/editor/parsers/run_options_parser.dart
create mode 100644
playground/frontend/test/modules/editor/parsers/run_options_parser_test.dart
create mode 100644
sdks/java/harness/src/main/java/org/apache/beam/fn/harness/Cache.java
create mode 100644
sdks/java/harness/src/main/java/org/apache/beam/fn/harness/Caches.java
create mode 100644
sdks/java/harness/src/test/java/org/apache/beam/fn/harness/CachesTest.java
