This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch
dependabot/npm_and_yarn/sdks/typescript/js-yaml-4.1.1
in repository https://gitbox.apache.org/repos/asf/beam.git
discard 07dd4d5ff45 Bump js-yaml from 4.1.0 to 4.1.1 in /sdks/typescript
add ae2d3b80a4c Bump golang.org/x/net from 0.46.0 to 0.47.0 in /sdks
(#36817)
add c8d7ca02867 Fix DinD Action failure (#36813)
add 57320861aa2 Ensure that Operations are aborted when MapTaskExecutor is
closed. Add tests around setup/teardown of DoFns (#36631)
add 715a482bf5a Update REVIEWERS.yml (#36828)
add 3af78a7a19d Fix DinD startup failures (#36835)
add d4f5372a67c Improve error message for decoding input stream (#36823)
add ce58563ffd9 update container tag (#36826)
add 53a0d3c31b7 Update BOM to 26.71.0 (#36663)
add 3721c17265e Adds proto fields for overwriting fetch/write state
functionality (#36708)
add 914c16eb1d1 Provide more contextual error message for PyCoder (#36825)
add b843f4af3eb Increase the GZip DeflaterOutputStream buffer size to
increase performance (#36818)
add a799d2b76ea Bump google.golang.org/grpc from 1.76.0 to 1.77.0 in /sdks
(#36843)
add 050369f39d1 make use of generic type info in *Utils classes
add b369dfa3814 make use of generic type info in *Utils classes (#36581)
add 87d204f6038 Fix inconsistent data type in GenericRecord and AvroSchema
for AvroWriter (#36839)
add 6d9df8cbc81 Remove 3.9-specific branches in typehinting module (#36732)
add eadbc6e74fc Kerberos auth python (#36211)
add f41cbde78c1 Allow users to specify trusted Avro serializable classes
to Dataflow worker (#36809)
add 534e1cb0dbf Add artifactregistry.writer to service account (#36756)
add 0ab2d039f8d Add @Pure annotations to Preconditions
add e84b318ad3c Add @Pure annotations to DoFnSignature
add a33d59420c7 Add @Pure annotations to DoFnSchemaInformation
add 8f5bc796213 Add @Pure annotations to TimerSpec
add edad6545dad Add @Pure annotations to SideInputReader
add 6c99f7ceb71 Add @Pure annotations to LateDataUtils
add f07ca33e355 Fix nullness errors in SimpleDoFnRunner and DoFnInvoker
add 87530a369fb Trivial change to KafkaSourceConsumerFnTest to eliminate
warning
add be65cfe8d78 Fix nullness of currentRecordId and currentRecordOffset
add 2a06694cc5e Merge pull request #35029: Fix nullness errors in
SimpleDoFnRunner and DoFnInvoker
add f5b4b6d2429 Add back sys import. (#36862)
add 5806b6e4b93 Support Beam MicrosInstant conversion to Avro Timestamp
(#36605)
add 614e742a9d0 Bump com.gradle.common-custom-user-data-gradle-plugin
(#36856)
add 108225d3797 Add Iceberg CDC support to YAML (#36641)
add 8e8fd1aefc1 Bump golang.org/x/crypto from 0.35.0 to 0.45.0 in
/.test-infra/mock-apis (#36865)
add e336419205c Bump github.com/aws/aws-sdk-go-v2/feature/s3/manager in
/sdks (#36867)
add 8ef3a2446bc Change KafkaIO to default to offset-based deduplication
when redistribute is enabled for Dataflow java runner. (#36849)
add 3d5d04b9f39 Enable some advanced state validates runner tests for
dataflow legacy and v2. (#36808)
add b0b67bf0796 Bump github.com/aws/aws-sdk-go-v2/service/s3 in /sdks
(#36872)
add 62ffc638bcb Fix DROP IF NOT EXISTS statement
add 4970cba9482 Merge pull request #36863: Fix DROP IF NOT EXISTS statement
add 7f499788a40 Add Iceberg Schema Support for PassThroughLogicalType
(#36870)
add b20ccbfe3ef [2/3] sdks/python: sink data with Milvus Search I/O
connector (#36729)
add 6d94d5c00ee Bump github.com/aws/aws-sdk-go-v2/feature/s3/manager in
/sdks (#36882)
add f9f13c331dd enhance python tests (#36852)
add 6a8244815b0 Move Kafka read with redistribute override to Kafka IO
package. (#36887)
add 94336fa0187 Make RestrictionTrackers.getProgress unblocking until
initial progress successfully returned (#36750)
add c84ccd385c2 callStateMap was accessed from multiple threads without
synchronization. changing callStateMap to concurrent hashmap (#36886)
add c8aa39898cf Fix GcsEndpoint serialization in GcsOptions (#36879)
add 876cda39434 [Docs] Add table of contents for contributor docs (#36861)
add 543056a7898 split redis into extra (#36772)
add 388e5e59ff6 Supports Asynchronous Runs in Interactive Beam (#36853)
add a9e2e68e507 [OpenTelemetry] Add OpenTelemetry options to
SdkHarnessOptions (#36716)
add 45911229938 Iceberg direct write (#36720)
add 203f45cee86 Support beam:logical_type:micros_instant:v1 in SpannerIo.
(#36840)
add db09899eb20 Move to macos-15 (#36900)
add d421c98075c Upgrade HCatalogIO to Hive 4.0.1 (#36901)
add 5b8743b9366 [BEAM-36736] Add state sampling for timer processing in
the Python SDK (#36737)
add 2f75792d044 Flink 1.20 support (#36893)
add 436662293b6 Bump github.com/nats-io/nats-server/v2 from 2.12.1 to
2.12.2 in /sdks (#36816)
add c72970d0ee3 Fix CSP issues (#36902)
add baa1751d837 comment out potential race condition until further
investigation in enrichment test (#36907)
add 744275030ac Moving to 2.71.0-SNAPSHOT on master branch.
add bdd042509b6 Update CHANGES.md to have fields for 2.71.0 release
add 2ffd443b986 Fix inconsistent handling of Firestore Project and
Database ID (#36895)
add 80ba916e94d fix retry scenario for query to table materialization
(#36912)
add 7323fb82cb5 split hdfs into extra (#36773)
add dde8937c124 Update CHANGES to mention extras changes (#36875)
add 31612929b02 Re-enable "PostCommit XVR GoUsingJava Dataflow" in cron
(#36909)
add a6a8613fca3 Bump github.com/aws/aws-sdk-go-v2/feature/s3/manager in
/sdks (#36922)
add d9c1e4e4f6c Bump cloud.google.com/go/storage from 1.57.1 to 1.57.2 in
/sdks (#36844)
add 782a0e7cc26 Revert "Fix CSP issues (#36902)" (#36929)
add 1fbd111b27f Bump golang.org/x/crypto from 0.44.0 to 0.45.0 in /sdks
(#36866)
add 43fdd50789d Fix runQuickstartJavaFlinkLocal (#36946)
add dbf4e5a32e9 Update BOM to 26.72.0 (#36919)
add e96c8dda71e Remove dead code. (#36943)
add efed19a6cd0 Fix typo in WriteToKafka docstring: deserializer ->
serializer (#36934)
add 0fb3736df78 Enforce version upper bound for scikit-learn. (#36948)
add fa1dad91e53 Fix linkage check script to handle detached HEAD state
add c490d2dbf61 Merge pull request #36924: Fix linkage check script to
handle detached HEAD state
add 0592f4fe566 Reflect that dill is no longer a default pickler. (#36903)
add bb8373f1abc Enforce hypothesis version upper bound (#36952)
add f242a88382f Include Beam version details in MongoDB handshake (#36949)
add 1cf077fc957 Bump google.golang.org/api from 0.255.0 to 0.256.0 in
/sdks (#36803)
add cbc31be7bbc Reuse secret instead of creating every time (#36955)
add 81a838cd15c propagate offset and record in output builder
add ead63ad5c07 Merge pull request #36926: Offset deduplication -
Propagate offset and record in output builder
add 81bb5066bab [Java] Dataflow runner v1 - Propagate drain mode (#36534)
add 79ea2e8562c Add support for sharding while avro write (#36933)
add 21109328f72 [Java] Fix CassandraIO ReadFn to quote column names for
reserved keywords (#36459)
add e3afe6207d7 Add new method of generating key for GBEK (#36891)
add b6556b4777b Fix reference leak in
ReifyTimestampAndWindowsParDoFnFactory (#36968)
add 67d5477d57f [Elasticsearch] Add support for v9.x (#36670)
add 554add18a2f Finish including Beam version details in MongoDB handshake
(#36961)
add 7ca164cc32f Bump google.golang.org/api from 0.256.0 to 0.257.0 in
/sdks (#36974)
add 0f7a189705c Improve coder byte calculation methods (#36830)
add ff954f98b29 Avoid creating Duration objects when direclty converting
to millis (#36969)
add d40da985458 Remove python 3.9
add d137814862a Add nvidia license
add 065a3aea428 Upgrade pip-licenses
add bfea1e36b30 Remove nvidia requirements
add 5cd9829766c Merge pull request #36954 from apache/fix-push-images
add abe649a8044 Enables persistent state validates runner tests for
Dataflow runners. (#36889)
add 54dcb0a4737 Bump github.com/aws/aws-sdk-go-v2 from 1.40.0 to 1.40.1 in
/sdks (#36973)
add fd9fe79bd4b Revert "Include Beam version details in MongoDB handshake
(#36949)"
add eb9e5b4327c add back import
add 660f061ac7d Merge pull request #36970 from
apache/revert-36949-add-metadata
add c9d2e60e676 Bump github.com/aws/aws-sdk-go-v2/feature/s3/manager in
/sdks (#36972)
add 712172dce5a Add arrow conversion support for timestamp nanoseconds
logical type. (#36960)
add ce76f81fbbc Add timestamp-nanos avro logical type support in bigquery
avro utils. (#36892)
add fc673aae5d8 Reapply CSP fix and correct new js and css references
(#36980)
add e24a2a2eeb9 Fix flaky
AsyncBatchWriteHandlerTest.correctlyLimitConcurrency (#36975)
add c0a58953470 Fix zero key length panic in Prism (#36983)
add efb37195348 Fix python postcommit
add 82dea870b92 Merge pull request #36977 from apache/users/damccorm/hdfsIT
add b59f3d93ef3 Bump github.com/spf13/cobra from 1.10.1 to 1.10.2 in /sdks
(#36987)
add ece2beb0659 Java Native Remote Inference (#36623)
add 1831cbaae95 Fix data race in test. (#36992)
add 9bcae0d5e03 Use HashMap.computeIfAbsent instead of get/put (#36990)
add fc31402d928 Attempt to fix flaky test by not requiring advance to
succeed immediately (#36976)
add 7b4a4c9e6f1 Update Python Dependencies (#36994)
add 3ce2abdc505 Bump jws from 4.0.0 to 4.0.1 in /sdks/typescript (#36996)
add 14dcd200f49 Bump js-yaml from 4.1.0 to 4.1.1 in /sdks/typescript
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (07dd4d5ff45)
\
N -- N -- N
refs/heads/dependabot/npm_and_yarn/sdks/typescript/js-yaml-4.1.1 (14dcd200f49)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.asf.yaml | 1 +
.github/REVIEWERS.yml | 7 +-
.github/actions/dind-up-action/action.yml | 24 +-
.../test-properties.json | 6 +-
...m_PostCommit_Java_ValidatesRunner_Dataflow.json | 7 +-
...it_Java_ValidatesRunner_Dataflow_Streaming.json | 8 +-
...ostCommit_Java_ValidatesRunner_Dataflow_V2.json | 8 +-
...Java_ValidatesRunner_Dataflow_V2_Streaming.json | 7 +-
.github/trigger_files/beam_PostCommit_Python.json | 2 +-
...on => beam_PostCommit_Python_MongoDBIO_IT.json} | 0
.../beam_PostCommit_Python_Xlang_Gcp_Direct.json | 2 +-
.../beam_PostCommit_XVR_GoUsingJava_Dataflow.json | 4 +-
.../workflows/beam_LoadTests_Java_GBK_Smoke.yml | 2 +-
.../beam_PostCommit_Java_Examples_Flink.yml | 2 +-
.../beam_PostCommit_Java_Nexmark_Flink.yml | 2 +-
.../beam_PostCommit_Java_PVR_Flink_Streaming.yml | 2 +-
.../workflows/beam_PostCommit_Java_Tpcds_Flink.yml | 2 +-
.../beam_PostCommit_Java_ValidatesRunner_Flink.yml | 2 +-
...PostCommit_Java_ValidatesRunner_Flink_Java8.yml | 2 +-
.github/workflows/beam_PostCommit_XVR_Flink.yml | 2 +-
.../beam_PostCommit_XVR_GoUsingJava_Dataflow.yml | 8 +-
...beam_PreCommit_Java_ElasticSearch_IO_Direct.yml | 3 +-
.../beam_PreCommit_Java_HCatalog_IO_Direct.yml | 15 -
.../workflows/beam_PreCommit_Java_IOs_Direct.yml | 15 -
.../beam_PreCommit_Java_PVR_Flink_Batch.yml | 2 +-
.../beam_PreCommit_Java_PVR_Flink_Docker.yml | 2 +-
.../workflows/beam_PreCommit_Python_Coverage.yml | 2 +
.github/workflows/build_release_candidate.yml | 1 -
.github/workflows/build_wheels.yml | 4 +-
.../republish_released_docker_containers.yml | 2 +-
.../run_rc_validation_java_quickstart.yml | 2 +-
.test-infra/mock-apis/go.mod | 10 +-
.test-infra/mock-apis/go.sum | 20 +-
CHANGES.md | 33 +-
build.gradle.kts | 3 +
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 27 +-
contributor-docs/README.md | 36 ++
examples/java/build.gradle | 1 +
gradle.properties | 6 +-
infra/iam/users.yml | 1 +
.../beam/model/fn_execution/v1/beam_fn_api.proto | 9 +
release/build.gradle.kts | 2 +-
release/src/main/scripts/run_rc_validation.sh | 8 +-
.../apache/beam/runners/core/LateDataUtils.java | 3 +
...TimeBoundedSplittableProcessElementInvoker.java | 4 +-
.../apache/beam/runners/core/SideInputReader.java | 13 +-
.../apache/beam/runners/core/SimpleDoFnRunner.java | 169 ++++--
.../wrappers/streaming/DoFnOperator.java | 0
runners/flink/{1.17 => 1.20}/build.gradle | 4 +-
.../job-server-container/build.gradle | 0
.../flink/{1.19 => 1.20}/job-server/build.gradle | 2 +-
.../wrappers/streaming/DoFnOperator.java | 8 +-
runners/flink/flink_runner.gradle | 5 +
.../translation/types/CoderTypeSerializer.java | 0
.../flink/FlinkExecutionEnvironmentsTest.java | 13 +-
.../runners/flink/ReadSourceStreamingTest.java | 7 +-
.../flink/streaming/GroupByNullKeyTest.java | 7 +-
.../flink/streaming/TopWikipediaSessionsTest.java | 7 +-
runners/google-cloud-dataflow-java/build.gradle | 37 +-
.../beam/runners/dataflow/DataflowRunner.java | 1 +
.../dataflow/worker/DataflowExecutionContext.java | 6 +-
.../worker/IntrinsicMapTaskExecutorFactory.java | 53 +-
.../ReifyTimestampAndWindowsParDoFnFactory.java | 8 +-
.../dataflow/worker/StreamingDataflowWorker.java | 2 +
.../worker/StreamingModeExecutionContext.java | 14 +-
.../dataflow/worker/UngroupedWindmillReader.java | 25 +-
.../dataflow/worker/WindmillKeyedWorkItem.java | 26 +-
.../beam/runners/dataflow/worker/WindmillSink.java | 14 +-
.../dataflow/worker/WindmillTimerInternals.java | 7 +-
.../dataflow/worker/WindowingWindmillReader.java | 3 +-
.../runners/dataflow/worker/streaming/Work.java | 14 +-
.../harness/SingleSourceWorkerHarness.java | 3 +
.../worker/util/common/worker/MapTaskExecutor.java | 71 ++-
.../client/grpc/GetWorkResponseChunkAssembler.java | 5 +-
.../client/grpc/GrpcDirectGetWorkStream.java | 1 +
.../windmill/client/grpc/GrpcGetWorkStream.java | 1 +
.../worker/windmill/work/WorkItemReceiver.java | 1 +
.../worker/windmill/work/WorkItemScheduler.java | 2 +
.../work/processing/StreamingWorkScheduler.java | 17 +-
.../dataflow/worker/FakeWindmillServer.java | 1 +
.../IntrinsicMapTaskExecutorFactoryTest.java | 137 ++++-
.../runners/dataflow/worker/SimpleParDoFnTest.java | 12 +-
.../worker/StreamingDataflowWorkerTest.java | 116 +++-
.../worker/StreamingGroupAlsoByWindowFnsTest.java | 2 +-
...reamingGroupAlsoByWindowsReshuffleDoFnTest.java | 2 +-
.../worker/StreamingModeExecutionContextTest.java | 1 +
.../dataflow/worker/WindmillKeyedWorkItemTest.java | 72 ++-
.../worker/WindmillTimerInternalsTest.java | 6 +-
.../dataflow/worker/WorkerCustomSourcesTest.java | 2 +
.../worker/streaming/ActiveWorkStateTest.java | 2 +
.../streaming/ComputationStateCacheTest.java | 1 +
.../FanOutStreamingEngineWorkerHarnessTest.java | 1 +
.../harness/WindmillStreamSenderTest.java | 1 +
.../worker/testing/TestCountingSource.java | 5 +
.../worker/util/BoundedQueueExecutorTest.java | 1 +
.../util/common/worker/MapTaskExecutorTest.java | 39 ++
.../StreamingApplianceWorkCommitterTest.java | 1 +
.../commits/StreamingEngineWorkCommitterTest.java | 1 +
.../client/grpc/GrpcDirectGetWorkStreamTest.java | 12 +-
.../client/grpc/GrpcWindmillServerTest.java | 6 +-
.../failures/WorkFailureProcessorTest.java | 1 +
.../work/refresh/ActiveWorkRefresherTest.java | 1 +
runners/prism/java/build.gradle | 7 +
scripts/beam-sql.sh | 2 +-
scripts/tools/bomupgrader.py | 1 +
sdks/go.mod | 98 ++--
sdks/go.sum | 213 +++----
sdks/go/examples/wasm/README.md | 2 +-
sdks/go/pkg/beam/core/core.go | 2 +-
sdks/go/pkg/beam/io/avroio/avroio.go | 118 +++-
sdks/go/pkg/beam/io/avroio/avroio_test.go | 108 +++-
.../prism/internal/engine/elementmanager.go | 11 +-
.../runners/prism/internal/engine/teststream.go | 9 +
sdks/go/pkg/beam/runners/prism/internal/execute.go | 4 +
.../beam/runners/prism/internal/handlerunner.go | 97 +++-
sdks/go/test/integration/integration.go | 3 +
sdks/java/build-tools/beam-linkage-check.sh | 7 +-
.../resources/beam/checkstyle/suppressions.xml | 1 +
sdks/java/container/boot.go | 22 +
.../container/license_scripts/dep_urls_java.yaml | 10 +-
sdks/java/core/build.gradle | 8 +
.../sdk/fn/splittabledofn/RestrictionTrackers.java | 94 ++-
.../java/org/apache/beam/sdk/io/Compression.java | 4 +-
.../apache/beam/sdk/options/SdkHarnessOptions.java | 40 ++
.../beam/sdk/schemas/logicaltypes/SqlTypes.java | 4 +
.../beam/sdk/schemas/logicaltypes/Timestamp.java | 1 -
.../beam/sdk/schemas/utils/AutoValueUtils.java | 16 +-
.../beam/sdk/schemas/utils/ByteBuddyUtils.java | 59 +-
.../beam/sdk/schemas/utils/JavaBeanUtils.java | 28 +-
.../apache/beam/sdk/schemas/utils/POJOUtils.java | 138 +----
.../java/org/apache/beam/sdk/state/TimerSpec.java | 2 +
.../java/org/apache/beam/sdk/transforms/DoFn.java | 4 +-
.../beam/sdk/transforms/DoFnSchemaInformation.java | 6 +
.../beam/sdk/transforms/reflect/DoFnInvoker.java | 28 +-
.../beam/sdk/transforms/reflect/DoFnSignature.java | 108 ++++
.../sdk/transforms/windowing/IntervalWindow.java | 3 +-
.../beam/sdk/transforms/windowing/PaneInfo.java | 17 +
.../beam/sdk/util/GcpHsmGeneratedSecret.java | 191 +++++++
.../org/apache/beam/sdk/util/Preconditions.java | 31 +
.../main/java/org/apache/beam/sdk/util/Secret.java | 45 +-
.../apache/beam/sdk/values/ValueWithRecordId.java | 17 +
.../org/apache/beam/sdk/values/WindowedValues.java | 8 +-
.../fn/splittabledofn/RestrictionTrackersTest.java | 90 ++-
.../beam/sdk/schemas/utils/POJOUtilsTest.java | 80 ---
.../sdk/transforms/GroupByEncryptedKeyTest.java | 80 +++
.../apache/beam/sdk/transforms/GroupByKeyIT.java | 112 ++++
.../beam/sdk/transforms/ParDoLifecycleTest.java | 11 +-
.../sdk/transforms/windowing/PaneInfoTest.java | 28 +
.../java/org/apache/beam/sdk/util/SecretTest.java | 18 +-
.../beam/sdk/extensions/arrow/ArrowConversion.java | 30 +-
.../sdk/extensions/arrow/ArrowConversionTest.java | 34 +-
.../extensions/avro/schemas/utils/AvroUtils.java | 71 +++
.../avro/schemas/utils/AvroUtilsTest.java | 117 ++++
.../sdk/extensions/gcp/options/GcsOptions.java | 1 -
sdks/java/extensions/ml/build.gradle | 1 +
sdks/java/extensions/sql/hcatalog/build.gradle | 4 +-
.../extensions/sql/impl/parser/SqlDropObject.java | 6 +-
.../io/aws2/common/AsyncBatchWriteHandlerTest.java | 2 +-
.../org/apache/beam/sdk/io/cassandra/ReadFn.java | 10 +
.../beam/sdk/io/cassandra/CassandraIOTest.java | 374 ++++++++++++
.../io/debezium/KafkaSourceConsumerFnTest.java | 2 +-
.../build.gradle | 8 +-
.../sdk/io/elasticsearch/ElasticsearchIOIT.java | 4 +-
.../sdk/io/elasticsearch/ElasticsearchIOTest.java | 4 +-
.../src/test/resources/clientkeystore | Bin
.../elasticsearch/ElasticsearchIOTestCommon.java | 15 +-
.../beam/sdk/io/elasticsearch/ElasticsearchIO.java | 2 +-
sdks/java/io/expansion-service/build.gradle | 2 +
.../sdk/io/gcp/bigquery/BigQueryAvroUtils.java | 106 +++-
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 2 +
.../gcp/bigquery/BigQueryStorageQuerySource.java | 22 +
...StorageApiDynamicDestinationsGenericRecord.java | 5 +-
.../providers/PortableBigQueryDestinations.java | 2 +-
.../sdk/io/gcp/firestore/FirestoreOptions.java | 5 +-
.../FirestoreStatefulComponentFactory.java | 23 +-
.../sdk/io/gcp/firestore/FirestoreV1WriteFn.java | 4 +-
.../beam/sdk/io/gcp/spanner/MutationUtils.java | 59 +-
.../beam/sdk/io/gcp/spanner/StructUtils.java | 37 +-
.../sdk/io/gcp/testing/FakeDatasetService.java | 4 +-
.../gcp/bigquery/BeamRowToStorageApiProtoTest.java | 5 +-
.../sdk/io/gcp/bigquery/BigQueryAvroUtilsTest.java | 170 +++++-
...gQueryFileLoadsSchemaTransformProviderTest.java | 23 +-
.../gcp/firestore/BaseFirestoreV1WriteFnTest.java | 8 +-
...storeV1FnBatchWriteWithDeadLetterQueueTest.java | 2 +-
.../FirestoreV1FnBatchWriteWithSummaryTest.java | 2 +-
.../sdk/io/gcp/firestore/it/BaseFirestoreIT.java | 3 +-
.../beam/sdk/io/gcp/spanner/MutationUtilsTest.java | 33 ++
.../beam/sdk/io/gcp/spanner/StructUtilsTest.java | 39 ++
sdks/java/io/hcatalog/build.gradle | 19 +-
.../apache/beam/sdk/io/hcatalog/HCatalogIO.java | 2 +-
.../io/hcatalog/test/EmbeddedMetastoreService.java | 17 +-
.../apache/beam/sdk/io/iceberg/BundleLifter.java | 170 ++++++
.../org/apache/beam/sdk/io/iceberg/IcebergIO.java | 19 +-
.../apache/beam/sdk/io/iceberg/IcebergUtils.java | 25 +-
.../IcebergWriteSchemaTransformProvider.java | 11 +
...owsToFiles.java => WriteDirectRowsToFiles.java} | 86 +--
.../sdk/io/iceberg/WriteGroupedRowsToFiles.java | 9 +-
.../beam/sdk/io/iceberg/WriteToDestinations.java | 173 +++++-
.../sdk/io/iceberg/WriteUngroupedRowsToFiles.java | 9 +-
.../beam/sdk/io/iceberg/BundleLifterTest.java | 99 ++++
.../beam/sdk/io/iceberg/IcebergUtilsTest.java | 103 ++++
.../java/org/apache/beam/sdk/io/jms/JmsIOTest.java | 28 +-
.../java/org/apache/beam/sdk/io/kafka/KafkaIO.java | 47 ++
.../kafka/KafkaReadWithRedistributeOverride.java | 73 +++
.../KafkaReadWithRedistributeOverrideTest.java | 133 +++++
.../jackson => ml/inference/openai}/build.gradle | 26 +-
.../ml/inference/openai/OpenAIModelHandler.java | 166 ++++++
.../sdk/ml/inference/openai/OpenAIModelInput.java | 63 ++
.../ml/inference/openai/OpenAIModelParameters.java | 114 ++++
.../ml/inference/openai/OpenAIModelResponse.java | 61 ++
.../ml/inference/openai/OpenAIModelHandlerIT.java | 402 +++++++++++++
.../inference/openai/OpenAIModelHandlerTest.java | 450 +++++++++++++++
.../inference/remote}/build.gradle | 26 +-
.../beam/sdk/ml/inference/remote/BaseInput.java} | 16 +-
.../sdk/ml/inference/remote/BaseModelHandler.java | 69 +++
.../ml/inference/remote/BaseModelParameters.java} | 41 +-
.../sdk/ml/inference/remote/BaseResponse.java} | 16 +-
.../sdk/ml/inference/remote/PredictionResult.java | 53 ++
.../sdk/ml/inference/remote/RemoteInference.java | 170 ++++++
.../beam/sdk/ml/inference/remote/RetryHandler.java | 103 ++++
.../ml/inference/remote/RemoteInferenceTest.java | 598 +++++++++++++++++++
sdks/python/apache_beam/coders/coder_impl.py | 17 +-
.../io/external/xlang_bigqueryio_it_test.py | 35 +-
.../io/gcp/tests/xlang_spannerio_it_test.py | 129 +++--
sdks/python/apache_beam/io/hadoopfilesystem.py | 11 +-
.../python/apache_beam/io/hadoopfilesystem_test.py | 7 +
.../hdfs_integration_test/hdfs_integration_test.sh | 2 +-
sdks/python/apache_beam/io/kafka.py | 51 +-
sdks/python/apache_beam/io/mongodbio.py | 14 +
sdks/python/apache_beam/io/requestresponse.py | 11 +-
.../apache_beam/ml/rag/ingestion/milvus_search.py | 346 +++++++++++
.../ml/rag/ingestion/milvus_search_it_test.py | 635 +++++++++++++++++++++
.../ml/rag/ingestion/milvus_search_test.py | 123 ++++
.../ml/rag/ingestion/postgres_common.py | 37 ++
.../python/apache_beam/options/pipeline_options.py | 2 +-
.../apache_beam/runners/dataflow/internal/names.py | 3 +-
.../runners/interactive/interactive_beam.py | 99 +++-
.../runners/interactive/interactive_beam_test.py | 391 +++++++++++++
.../runners/interactive/interactive_environment.py | 19 +
.../interactive/interactive_environment_test.py | 41 ++
.../runners/interactive/recording_manager.py | 478 +++++++++++++++-
.../runners/interactive/recording_manager_test.py | 500 ++++++++++++++++
.../apache_beam/runners/interactive/utils_test.py | 12 +
.../apache_beam/runners/worker/bundle_processor.py | 6 +-
.../apache_beam/runners/worker/operations.pxd | 1 +
.../apache_beam/runners/worker/operations.py | 22 +-
.../runners/worker/statesampler_test.py | 185 ++++++
sdks/python/apache_beam/transforms/core_it_test.py | 57 ++
sdks/python/apache_beam/transforms/util.py | 160 +++++-
sdks/python/apache_beam/transforms/util_test.py | 129 ++++-
.../typehints/native_type_compatibility.py | 5 +-
.../apache_beam/typehints/trivial_inference.py | 8 +-
sdks/python/apache_beam/typehints/typehints.py | 6 +-
.../python/apache_beam/typehints/typehints_test.py | 12 +-
sdks/python/apache_beam/version.py | 2 +-
.../yaml/extended_tests/data/enrichment.yaml | 78 +--
.../yaml/extended_tests/databases/iceberg.yaml | 54 +-
sdks/python/apache_beam/yaml/standard_io.yaml | 28 +
sdks/python/conftest.py | 34 +-
sdks/python/container/Dockerfile | 2 +-
.../container/ml/py310/base_image_requirements.txt | 80 ++-
.../container/ml/py310/gpu_image_requirements.txt | 108 ++--
.../container/ml/py311/base_image_requirements.txt | 80 ++-
.../container/ml/py311/gpu_image_requirements.txt | 108 ++--
.../container/ml/py312/base_image_requirements.txt | 80 ++-
.../container/ml/py312/gpu_image_requirements.txt | 108 ++--
.../container/ml/py313/base_image_requirements.txt | 80 ++-
.../container/py310/base_image_requirements.txt | 66 +--
.../container/py311/base_image_requirements.txt | 64 +--
.../container/py312/base_image_requirements.txt | 64 +--
.../container/py313/base_image_requirements.txt | 64 +--
sdks/python/expansion-service-container/Dockerfile | 2 +-
sdks/python/setup.py | 11 +-
sdks/python/tox.ini | 7 +-
sdks/typescript/package-lock.json | 36 +-
sdks/typescript/package.json | 2 +-
sdks/typescript/src/apache_beam/runners/flink.ts | 2 +-
settings.gradle.kts | 73 ++-
.../site/assets/css/fontawesome/release-v5.4.1.css | 5 +
website/www/site/assets/css/roboto/roboto.css | 35 ++
.../site/assets/css/swiper@8/swiper-bundle.min.css | 13 +
.../www/site/assets/js/jquery/jquery-2.2.4.min.js | 4 +
.../sdks/python-pipeline-dependencies.md | 14 +-
website/www/site/layouts/partials/head.html | 12 +-
.../www/site/layouts/partials/head_homepage.html | 12 +-
website/www/site/static/.htaccess | 5 +-
286 files changed, 11005 insertions(+), 1786 deletions(-)
copy .github/trigger_files/{beam_PostCommit_Java_Examples_Dataflow_ARM.json =>
beam_PostCommit_Python_MongoDBIO_IT.json} (100%)
create mode 100644 contributor-docs/README.md
copy runners/flink/{ =>
1.17}/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java
(100%)
copy runners/flink/{1.17 => 1.20}/build.gradle (94%)
copy runners/flink/{1.17 => 1.20}/job-server-container/build.gradle (100%)
copy runners/flink/{1.19 => 1.20}/job-server/build.gradle (95%)
rename runners/flink/{ =>
1.20}/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java
(99%)
rename runners/flink/{1.17 =>
}/src/main/java/org/apache/beam/runners/flink/translation/types/CoderTypeSerializer.java
(100%)
create mode 100644
sdks/java/core/src/main/java/org/apache/beam/sdk/util/GcpHsmGeneratedSecret.java
copy sdks/java/io/elasticsearch-tests/{elasticsearch-tests-8 =>
elasticsearch-tests-9}/build.gradle (91%)
copy sdks/java/io/elasticsearch-tests/{elasticsearch-tests-8 =>
elasticsearch-tests-9}/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOIT.java
(99%)
copy sdks/java/io/elasticsearch-tests/{elasticsearch-tests-8 =>
elasticsearch-tests-9}/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java
(99%)
copy sdks/java/io/elasticsearch-tests/{elasticsearch-tests-7 =>
elasticsearch-tests-9}/src/test/resources/clientkeystore (100%)
create mode 100644
sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/BundleLifter.java
copy
sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/{WriteGroupedRowsToFiles.java
=> WriteDirectRowsToFiles.java} (58%)
create mode 100644
sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/BundleLifterTest.java
create mode 100644
sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaReadWithRedistributeOverride.java
create mode 100644
sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaReadWithRedistributeOverrideTest.java
copy sdks/java/{extensions/jackson => ml/inference/openai}/build.gradle (65%)
create mode 100644
sdks/java/ml/inference/openai/src/main/java/org/apache/beam/sdk/ml/inference/openai/OpenAIModelHandler.java
create mode 100644
sdks/java/ml/inference/openai/src/main/java/org/apache/beam/sdk/ml/inference/openai/OpenAIModelInput.java
create mode 100644
sdks/java/ml/inference/openai/src/main/java/org/apache/beam/sdk/ml/inference/openai/OpenAIModelParameters.java
create mode 100644
sdks/java/ml/inference/openai/src/main/java/org/apache/beam/sdk/ml/inference/openai/OpenAIModelResponse.java
create mode 100644
sdks/java/ml/inference/openai/src/test/java/org/apache/beam/sdk/ml/inference/openai/OpenAIModelHandlerIT.java
create mode 100644
sdks/java/ml/inference/openai/src/test/java/org/apache/beam/sdk/ml/inference/openai/OpenAIModelHandlerTest.java
copy sdks/java/{extensions/timeseries => ml/inference/remote}/build.gradle
(70%)
copy sdks/{python/test-suites/tox/py311/build.gradle =>
java/ml/inference/remote/src/main/java/org/apache/beam/sdk/ml/inference/remote/BaseInput.java}
(74%)
create mode 100644
sdks/java/ml/inference/remote/src/main/java/org/apache/beam/sdk/ml/inference/remote/BaseModelHandler.java
copy sdks/{python/test-suites/tox/pycommon/build.gradle =>
java/ml/inference/remote/src/main/java/org/apache/beam/sdk/ml/inference/remote/BaseModelParameters.java}
(50%)
copy
sdks/java/{io/splunk/src/main/java/org/apache/beam/sdk/io/splunk/package-info.java
=>
ml/inference/remote/src/main/java/org/apache/beam/sdk/ml/inference/remote/BaseResponse.java}
(66%)
create mode 100644
sdks/java/ml/inference/remote/src/main/java/org/apache/beam/sdk/ml/inference/remote/PredictionResult.java
create mode 100644
sdks/java/ml/inference/remote/src/main/java/org/apache/beam/sdk/ml/inference/remote/RemoteInference.java
create mode 100644
sdks/java/ml/inference/remote/src/main/java/org/apache/beam/sdk/ml/inference/remote/RetryHandler.java
create mode 100644
sdks/java/ml/inference/remote/src/test/java/org/apache/beam/sdk/ml/inference/remote/RemoteInferenceTest.java
create mode 100644 sdks/python/apache_beam/ml/rag/ingestion/milvus_search.py
create mode 100644
sdks/python/apache_beam/ml/rag/ingestion/milvus_search_it_test.py
create mode 100644
sdks/python/apache_beam/ml/rag/ingestion/milvus_search_test.py
create mode 100644 website/www/site/assets/css/fontawesome/release-v5.4.1.css
create mode 100644 website/www/site/assets/css/roboto/roboto.css
create mode 100644 website/www/site/assets/css/swiper@8/swiper-bundle.min.css
create mode 100644 website/www/site/assets/js/jquery/jquery-2.2.4.min.js
