This is an automated email from the ASF dual-hosted git repository.
github-actions[bot] pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from 9f117ff5d7d Bump github.com/moby/moby/api from 1.55.0 to 1.56.0 in
/sdks (#40037)
add f0a19da20a9 Bump github.com/moby/moby/client from 0.5.1 to 0.6.0 in
/sdks (#40054)
add f757785ba15 Bump cloud.google.com/go/bigtable from 1.53.0 to 1.56.0 in
/sdks (#40056)
add 737972ef463 Bump github.com/aws/aws-sdk-go-v2/service/s3 in /sdks
(#40053)
add 07ddef7a3b5 Bump github.com/aws/aws-sdk-go-v2/config from 1.33.2 to
1.33.3 in /sdks (#40057)
add 6053b781a2c [Java] Move GCP-related secret classes and tests from
sdks/java/core to google-cloud-platform-core (#40019)
add ac159f4f37f Bump org.codehaus.plexus:plexus-xml from 3.0.2 to 4.2.0
(#39857)
add c4cfdefa123 Bump docker/setup-qemu-action from 4.2.0 to 4.3.0 (#40012)
add b1068bc0acf AddFiles: SchemaEvolutionConfig, the public settings for
the schema pre-pass (#40003)
add 6a0bd90065a Fix new line (#40060)
add ef94119c2f3 Fix artifact staging windows filename (#39916)
add abe41005e7b [Spark][#36841] Add the DataSourceV2 unbounded source for
the Spark 4 streaming runner (#39971)
add bce7e07a9df Introduce LABEL_NOTICE for PR bot (#40002)
add d79a2487a87 normalize firestoreIO (#39941)
add 2bdcd3dea60 use default SDK image and aarch64 wheel (#40017)
add eb91b752071 Bump google.golang.org/grpc in
/learning/tour-of-beam/backend (#40065)
add e33cd105272 Bump nodemailer from 9.0.1 to 9.1.1 in
/scripts/ci/issue-report (#40066)
No new revisions were added by this update.
Summary of changes:
.agent/skills/contributing/SKILL.md | 7 +-
.github/autolabeler.yml | 21 +-
.../trigger_files/beam_PostCommit_Python_Arm.json | 2 +-
.../trigger_files/beam_PostCommit_XVR_Direct.json | 2 +-
.../trigger_files/beam_PostCommit_XVR_Flink.json | 5 +-
...am_PostCommit_XVR_JavaUsingPython_Dataflow.json | 5 +-
.../trigger_files/beam_PostCommit_XVR_Spark3.json | 4 +-
.../beam_PostCommit_Yaml_Xlang_Direct.json | 3 +-
.github/workflows/beam_PostCommit_Python_Arm.yml | 12 +-
.../beam_PostCommit_TransformService_Direct.yml | 4 +-
.../beam_PostCommit_Yaml_Xlang_Direct.yml | 2 +-
.../workflows/beam_PreCommit_Yaml_Xlang_Direct.yml | 2 +-
.github/workflows/build_wheels.yml | 2 +-
.github/workflows/java_tests.yml | 29 +-
buildSrc/build.gradle.kts | 2 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 6 +
learning/tour-of-beam/backend/go.mod | 71 +-
learning/tour-of-beam/backend/go.sum | 166 ++--
runners/java-fn-execution/build.gradle | 1 +
.../artifact/ArtifactStagingService.java | 19 +-
.../fnexecution/environment/ProcessManager.java | 3 +-
.../artifact/ArtifactStagingServiceTest.java | 41 +
.../environment/ProcessManagerTest.java | 57 +-
.../io/streaming/BeamReaderCache.java | 352 +++++++
.../io/streaming/BeamSourceCheckpoint.java | 227 +++++
.../io/streaming/UnboundedSourceDataset.java | 721 ++++++++++++++
.../io/streaming/BeamMicroBatchSourceTest.java | 1028 ++++++++++++++++++++
runners/spark/job-server/spark_job_server.gradle | 2 +
.../SparkStructuredStreamingPipelineOptions.java | 9 +
scripts/ci/issue-report/package-lock.json | 15 +-
scripts/ci/issue-report/package.json | 2 +-
scripts/ci/pr-bot/findPrsNeedingAttention.ts | 4 +-
scripts/ci/pr-bot/processNewPrs.ts | 4 +-
scripts/ci/pr-bot/shared/commentStrings.ts | 47 +-
scripts/ci/pr-bot/shared/userCommand.ts | 12 +-
scripts/ci/pr-bot/test/commentStringsTest.ts | 45 +
sdks/go.mod | 34 +-
sdks/go.sum | 68 +-
.../resources/beam/checkstyle/suppressions.xml | 8 +-
sdks/java/core/build.gradle | 13 -
.../sdk/transforms/GroupByEncryptedKeyTest.java | 153 +--
.../apache/beam/sdk/transforms/GroupByKeyTest.java | 112 ---
.../java/org/apache/beam/sdk/util/SecretTest.java | 161 ---
.../apache/beam/sdk/util/TestSecretRegistrar.java} | 8 +-
.../util/construction/ValidateRunnerXlangTest.java | 67 +-
.../google-cloud-platform-core/build.gradle | 6 +
.../gcp}/util/GcpHsmGeneratedSecret.java | 8 +-
.../gcp}/util/GcpHsmGeneratedSecretRegistrar.java | 3 +-
.../beam/sdk/extensions/gcp}/util/GcpSecret.java | 3 +-
.../extensions/gcp}/util/GcpSecretRegistrar.java | 3 +-
.../transforms/GcpGroupByEncryptedKeyTest.java} | 130 ++-
.../gcp/transforms/GcpGroupByKeyIT.java} | 11 +-
.../gcp}/util/GcpSecretRegistrarTest.java | 3 +-
.../sdk/extensions/gcp/util/GcpSecretTest.java} | 186 +---
sdks/java/io/components/build.gradle | 3 +
...FirestoreReadSchemaTransformConfiguration.java} | 50 +-
.../FirestoreReadSchemaTransformProvider.java | 216 ++++
.../beam/sdk/io/gcp/firestore/FirestoreUtils.java | 312 ++++++
.../beam/sdk/io/gcp/firestore/FirestoreV1.java | 7 +-
.../sdk/io/gcp/firestore/FirestoreV1ReadFn.java | 38 +-
...irestoreWriteSchemaTransformConfiguration.java} | 53 +-
.../FirestoreWriteSchemaTransformProvider.java | 201 ++++
.../FirestoreSchemaTransformProviderTest.java | 179 ++++
.../sdk/io/gcp/firestore/FirestoreUtilsTest.java | 65 ++
.../beam/sdk/io/iceberg/SchemaEvolutionConfig.java | 152 +++
.../beam/sdk/io/iceberg/SchemaEvolutionOption.java | 51 +
.../sdk/io/iceberg/SchemaEvolutionConfigTest.java | 92 ++
.../runners/portability/expansion_service_test.py | 6 +
.../yaml/extended_tests/databases/firestore.yaml | 84 ++
sdks/python/apache_beam/yaml/integration_tests.py | 54 +
sdks/python/apache_beam/yaml/standard_io.yaml | 27 +
sdks/python/build.gradle | 13 +-
sdks/python/scripts/run_integration_test.sh | 7 +-
sdks/python/setup.py | 4 +
sdks/python/test-suites/dataflow/common.gradle | 14 +-
75 files changed, 4480 insertions(+), 1059 deletions(-)
create mode 100644
runners/spark/4/src/main/java/org/apache/beam/runners/spark/structuredstreaming/io/streaming/BeamReaderCache.java
create mode 100644
runners/spark/4/src/main/java/org/apache/beam/runners/spark/structuredstreaming/io/streaming/BeamSourceCheckpoint.java
create mode 100644
runners/spark/4/src/main/java/org/apache/beam/runners/spark/structuredstreaming/io/streaming/UnboundedSourceDataset.java
create mode 100644
runners/spark/4/src/test/java/org/apache/beam/runners/spark/structuredstreaming/io/streaming/BeamMicroBatchSourceTest.java
create mode 100644 scripts/ci/pr-bot/test/commentStringsTest.ts
copy
sdks/java/core/src/{main/java/org/apache/beam/sdk/util/GcpSecretRegistrar.java
=> test/java/org/apache/beam/sdk/util/TestSecretRegistrar.java} (80%)
rename sdks/java/{core/src/main/java/org/apache/beam/sdk =>
extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp}/util/GcpHsmGeneratedSecret.java
(96%)
rename sdks/java/{core/src/main/java/org/apache/beam/sdk =>
extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp}/util/GcpHsmGeneratedSecretRegistrar.java
(93%)
rename sdks/java/{core/src/main/java/org/apache/beam/sdk =>
extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp}/util/GcpSecret.java
(98%)
rename sdks/java/{core/src/main/java/org/apache/beam/sdk =>
extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp}/util/GcpSecretRegistrar.java
(93%)
copy
sdks/java/{core/src/test/java/org/apache/beam/sdk/transforms/GroupByEncryptedKeyTest.java
=>
extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/transforms/GcpGroupByEncryptedKeyTest.java}
(80%)
rename
sdks/java/{core/src/test/java/org/apache/beam/sdk/transforms/GroupByKeyIT.java
=>
extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/transforms/GcpGroupByKeyIT.java}
(97%)
rename sdks/java/{core/src/test/java/org/apache/beam/sdk =>
extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp}/util/GcpSecretRegistrarTest.java
(96%)
copy sdks/java/{core/src/test/java/org/apache/beam/sdk/util/SecretTest.java =>
extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/util/GcpSecretTest.java}
(62%)
copy
sdks/java/io/{mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/MongoDbReadSchemaTransformConfiguration.java
=>
google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreReadSchemaTransformConfiguration.java}
(57%)
create mode 100644
sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreReadSchemaTransformProvider.java
create mode 100644
sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreUtils.java
copy
sdks/java/io/{mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/MongoDbWriteSchemaTransformConfiguration.java
=>
google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreWriteSchemaTransformConfiguration.java}
(51%)
create mode 100644
sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreWriteSchemaTransformProvider.java
create mode 100644
sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreSchemaTransformProviderTest.java
create mode 100644
sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreUtilsTest.java
create mode 100644
sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/SchemaEvolutionConfig.java
create mode 100644
sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/SchemaEvolutionOption.java
create mode 100644
sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/SchemaEvolutionConfigTest.java
create mode 100644
sdks/python/apache_beam/yaml/extended_tests/databases/firestore.yaml
