This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git.
from c4fc398 Merge pull request #16368: [BEAM-13509] Stop sharing SQS
client between readers of same source.
add 17b62ad Merge pull request #16404: [BEAM-13586] Fix NPE in
DefaultS3ClientBuilderFactory
add 36ce02b [BEAM-13480] Sickbay
PubSubIntegrationTest.test_streaming_data_only on Dataflow (#16255)
add 3d272e7 Fix sdk_container_builder too many values to unpack error
add 1636a3a Merge pull request #16417 from y1chi/python_val
add 43a17a3 [BEAM-13430] Swap to use "mainClass" instead of "main" since
it was deprecated. (#16400)
add e6c73b4 [BEAM-13430] Replace deprecated "appendix" with
"archiveAppendix" (#16401)
add 061533c [BEAM-13591] Bump log4j2 version to 2.17.1
add 5a11778 Merge pull request #16415 from deadwind4/BEAM-13591
add 122ae34 [BEAM-13015] Add jamm as a java agent to the Java SDK harness
container (#16412)
add 2d4a17c [BEAM-13430] Partially revert
https://github.com/apache/beam/commit/eaacf709a861ea1c260e3856b720266348b1aecc
(#16419)
add 7b5c4cd [BEAM-12391] update avro sink to close the opened file
handle, instead of getting it by reference from the writer. This ensures that
when we load the python writer implementation we don't run into an attribute
error
add f63451e Merge pull request #16246: [BEAM-12391] update avro sink to
close the opened file handle directly
add 6045ee1 [BEAM-13482] Python fully qualified name external transforms.
add d13c21f Add Python expansion service entry point.
add 0c2f5a5 Merge pull request #16266 [BEAM-13482] Python fully qualified
name external transforms.
add 818428f Merge pull request #15863 from [BEAM-13184] Autosharding for
JdbcIO.write* transforms
add 9d2fcf3 Add Flink runner support for OrderedListState. This version
reads the list entirely into memory, so will not work well for large lists.
add b1341bc Merge pull request #15984: [BEAM-2791] OrderedListState for
Flink
add 3220631 [BEAM-11936] Enable FloatingPointAssertionWithinEpsilon
errorprone check (#16261)
add 9973c4f [BEAM-11936] Enable LockNotBeforeTry errorprone check (#16259)
add 7eeffe0 [BEAM-11936] Enable errorprone unused checks (#16262)
add 6b68536 Add Nexmark Query 14 (#16337)
add 83e9218 [BEAM-13569] Change Spark dependencies to implementation.
add f0289c0 remove redundant dependency
add e35eea8 remove redundant testImplementation dependencies
add 4c9d4f0 Merge pull request #16396 from ibzib/BEAM-13569
add ea65a05 [BEAM-13015] Migrate all user state and side implementations
to support caching. (#16263)
add eea07cf Fix remaining failing perf IT tests.
add 5ccb002 Merge pull request #16418 from y1chi/perf-it-2
add b82e407 [BEAM-13459] Update CHANGES.md, add note about artifact
caching for python jobs
add 56c2082 [BEAM-13459] Update CHANGES.md, add note about artifact
caching python jobs #16416
No new revisions were added by this update.
Summary of changes:
...Java_PortableValidatesRunner_Spark_Batch.groovy | 2 +
...va_PortableValidatesRunner_Flink_Docker.groovy} | 10 +-
.test-infra/validate-runner/build.gradle | 2 +-
CHANGES.md | 1 +
build.gradle.kts | 2 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 19 +-
examples/java/build.gradle | 4 +-
.../datatokenization/DataTokenization.java | 2 +-
.../examples/complete/datatokenization/README.md | 2 +-
.../complete/kafkatopubsub/KafkaToPubsub.java | 2 +-
.../beam/examples/complete/kafkatopubsub/README.md | 2 +-
.../apache/beam/examples/snippets/Snippets.java | 8 +
examples/java/twitter/build.gradle | 2 +-
.../core/construction/PTransformMatchersTest.java | 21 -
runners/direct-java/build.gradle | 4 +
.../runners/direct/TransformEvaluatorRegistry.java | 16 -
.../runners/direct/CloningBundleFactoryTest.java | 67 ---
runners/flink/flink_runner.gradle | 5 +-
runners/flink/job-server/flink_job_server.gradle | 47 +-
.../flink/FlinkStreamingTransformTranslators.java | 17 -
.../wrappers/streaming/DoFnOperator.java | 2 +-
.../state/FlinkBroadcastStateInternals.java | 131 ------
.../streaming/state/FlinkStateInternals.java | 153 +++++-
.../streaming/ExecutableStageDoFnOperatorTest.java | 1 +
runners/google-cloud-dataflow-java/build.gradle | 4 +-
.../beam/runners/dataflow/DataflowRunner.java | 110 -----
.../runners/dataflow/DataflowPipelineJobTest.java | 25 -
.../fn/data/RemoteGrpcPortWriteOperation.java | 6 +-
.../control/DefaultJobBundleFactory.java | 12 +-
.../fnexecution/control/RemoteExecutionTest.java | 2 -
runners/jet/build.gradle | 2 +
runners/portability/java/build.gradle | 2 +
runners/samza/build.gradle | 2 +
runners/samza/job-server/build.gradle | 45 +-
runners/spark/job-server/spark_job_server.gradle | 192 ++++----
runners/spark/spark_runner.gradle | 18 +-
runners/twister2/build.gradle | 2 +
sdks/java/container/Dockerfile | 3 +
sdks/java/container/boot.go | 9 +
sdks/java/container/build.gradle | 1 +
sdks/java/container/common.gradle | 1 +
sdks/java/core/build.gradle | 1 +
.../org/apache/beam/sdk/schemas/SchemaCoder.java | 20 -
...Metrics.java => UsesSdkHarnessEnvironment.java} | 5 +-
.../apache/beam/sdk/values/PCollectionViews.java | 59 ---
.../apache/beam/sdk/SdkHarnessEnvironmentTest.java | 69 +++
.../apache/beam/sdk/coders/CoderRegistryTest.java | 4 -
.../org/apache/beam/sdk/transforms/ParDoTest.java | 15 +-
.../sdk/transforms/reflect/DoFnSignaturesTest.java | 9 -
.../GrowableOffsetRangeTrackerTest.java | 2 +-
sdks/java/expansion-service/app/build.gradle | 2 +-
sdks/java/expansion-service/build.gradle | 2 +-
.../core/translate/TimestampExtractTransform.java | 8 -
.../schemaio-expansion-service/build.gradle | 2 +-
sdks/java/extensions/sql/build.gradle | 4 +-
sdks/java/extensions/sql/datacatalog/build.gradle | 2 +-
.../extensions/sql/expansion-service/build.gradle | 2 +-
sdks/java/extensions/sql/hcatalog/build.gradle | 2 +-
.../org/apache/beam/sdk/fn/CancellableQueue.java | 4 +-
.../beam/sdk/fn/stream/PrefetchableIterable.java | 3 +
.../beam/sdk/fn/stream/PrefetchableIterables.java | 53 ++-
.../sdk/fn/stream/PrefetchableIterablesTest.java | 20 +
sdks/java/harness/build.gradle | 6 +-
.../java/org/apache/beam/fn/harness/Caches.java | 1 +
.../apache/beam/fn/harness/state/BagUserState.java | 57 ++-
.../beam/fn/harness/state/FnApiStateAccessor.java | 135 +++---
.../beam/fn/harness/state/IterableSideInput.java | 47 +-
.../state/LazyCachingIteratorToIterable.java | 6 +-
.../beam/fn/harness/state/MultimapSideInput.java | 69 +--
.../beam/fn/harness/state/MultimapUserState.java | 184 +++++---
.../fn/harness/state/StateFetchingIterators.java | 9 +-
.../beam/fn/harness/state/BagUserStateTest.java | 170 ++++++-
.../fn/harness/state/IterableSideInputTest.java | 96 ++++
.../fn/harness/state/MultimapSideInputTest.java | 71 ++-
.../fn/harness/state/MultimapUserStateTest.java | 524 +++++++++++++++++----
.../harness/state/StateFetchingIteratorsTest.java | 2 +-
.../io/aws2/s3/DefaultS3ClientBuilderFactory.java | 27 +-
.../aws2/s3/DefaultS3ClientBuilderFactoryTest.java | 92 ++++
.../elasticsearch-tests-5/build.gradle | 2 +-
.../elasticsearch-tests-6/build.gradle | 2 +-
.../elasticsearch-tests-7/build.gradle | 2 +-
.../elasticsearch-tests-common/build.gradle | 2 +-
sdks/java/io/expansion-service/build.gradle | 2 +-
.../beam/sdk/io/gcp/spanner/SpannerAccessor.java | 24 -
sdks/java/io/hadoop-format/build.gradle | 3 +
.../sdk/io/hadoop/format/TestRowDBWritable.java | 10 -
sdks/java/io/hcatalog/build.gradle | 2 +-
.../java/org/apache/beam/sdk/io/jdbc/JdbcIO.java | 107 ++++-
.../java/org/apache/beam/sdk/io/jdbc/JdbcIOIT.java | 39 ++
.../org/apache/beam/sdk/io/jdbc/JdbcIOTest.java | 30 ++
.../org/apache/beam/sdk/io/xml/XmlSourceTest.java | 4 +-
sdks/java/testing/expansion-service/build.gradle | 4 +-
sdks/java/testing/kafka-service/build.gradle | 4 +-
sdks/java/testing/load-tests/build.gradle | 3 +-
sdks/java/testing/nexmark/build.gradle | 2 +-
.../apache/beam/sdk/nexmark/NexmarkLauncher.java | 3 +
.../apache/beam/sdk/nexmark/NexmarkQueryName.java | 1 +
.../nexmark/queries/{Query9.java => Query14.java} | 22 +-
sdks/java/testing/tpcds/build.gradle | 2 +-
sdks/java/testing/watermarks/build.gradle | 2 +-
sdks/python/apache_beam/io/avroio.py | 25 +-
sdks/python/apache_beam/io/avroio_test.py | 15 +
.../apache_beam/io/gcp/pubsub_integration_test.py | 2 +
.../runners/portability/expansion_service_main.py | 72 +++
.../runners/portability/sdk_container_builder.py | 2 +-
sdks/python/apache_beam/transforms/external.py | 10 +-
.../transforms/fully_qualified_named_transform.py | 107 +++++
.../fully_qualified_named_transform_test.py | 102 ++++
sdks/python/apache_beam/typehints/schemas.py | 21 +-
.../site/content/en/get-started/quickstart-java.md | 2 +-
110 files changed, 2226 insertions(+), 1146 deletions(-)
copy
.test-infra/jenkins/{job_PreCommit_Java_PortableValidatesRunner_Flink_Batch.groovy
=> job_PreCommit_Java_PortableValidatesRunner_Flink_Docker.groovy} (79%)
copy
sdks/java/core/src/main/java/org/apache/beam/sdk/testing/{UsesSystemMetrics.java
=> UsesSdkHarnessEnvironment.java} (86%)
create mode 100644
sdks/java/core/src/test/java/org/apache/beam/sdk/SdkHarnessEnvironmentTest.java
create mode 100644
sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/IterableSideInputTest.java
create mode 100644
sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/s3/DefaultS3ClientBuilderFactoryTest.java
copy
sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/{Query9.java
=> Query14.java} (63%)
create mode 100644
sdks/python/apache_beam/runners/portability/expansion_service_main.py
create mode 100644
sdks/python/apache_beam/transforms/fully_qualified_named_transform.py
create mode 100644
sdks/python/apache_beam/transforms/fully_qualified_named_transform_test.py
