This is an automated email from the ASF dual-hosted git repository.
ahmedabu98 pushed a change to branch sql-shell-cancel
in repository https://gitbox.apache.org/repos/asf/beam.git
from 9a56e458381 enable canceling SQL streaming pipeline
add 86504f71ce9 Bump google.golang.org/api from 0.301.0 to 0.302.0 in
/sdks (#40627)
add 0072521920c [Java] Fix GRPC channel leak (#39663)
add 3336b153784 [Prism] Guard ClearOrderedListState against uninitialized
state maps (#40626)
add f3a5f4f1542 Add stale GCS test bucket cleaner (#40386)
add 6572166744e [Java] Make non-extensible classes final instead of
suppressing CT_CONSTRUCTOR_THROW (#40444)
add 1f76c9c36d6 Fix Python unit tests broken by google-api-core arriving
via cloud-sql-python-connector 1.23.0 (#40623)
add 084fb321e57 Restore Java 11 support in setJavaVerOptions (#40634)
add d5f6472f73a Enable throttling logic in RemoteInference. (#40598)
add 964d15b47a4 [Python] Document the import-time default of
PeriodicImpulse start_timestamp (#40599)
add a3f3c8e01d4 Fix handling of cancelled tickets. (#40588)
add 1fc615fcd2b [typescript] - fix typescript failures (#40635)
add a17e73a0da4 Clarify known restrictions for model updates. (#40600)
add a74f328f26e [yaml] - fix post commit yaml test (#40633)
add e41f0ce1d1d [Python] Add examples for AWS credentials from a secret
manager (#40592)
add 8eda31b281f [Python] Fix max_models_per_worker_hint not being enforced
in KeyedModelHandler (#40500)
add 2be52540457 [Prism] Finalize bundles of DoFns fused after another
transform (#40640)
add 8ef717637e9 Bump minimum cryptography bounds beyond known severe CVEs
(#40591)
add 40df757ea39 Support attributing BigQuery quota and billing to a custom
GCP project (Python, Java, Go) (#39203)
add 707b951e5e0 Preserve HTTP errors when quota response is not JSON
(#40631)
add 5fc82a67af6 [Spanner Change Streams] Match single-RPC deadline to
total timeout in SpannerIO.ReadChangeStream (#40616)
add 10d3b8d316d Fix temp table cleanup for continuous read in BigQueryIO
(#37535)
add 49d41adc895 Bump google.golang.org/api from 0.302.0 to 0.304.0 in
/sdks (#40650)
add e4bc725b42d Bump zizmorcore/zizmor-action from 0.6.4 to 0.7.1 (#40652)
add 5fa8c2c878e Bump github/codeql-action from 4.38.3 to 4.39.0 (#40651)
add 8648b891523 Update users.yml (#40654)
add 590abac1145 [Python] Avoid ZeroDivisionError in ConcatRangeTracker for
zero weights (#40549)
add 14575dcd143 Update api key for tour of beam (#40655)
add 2c60b1996e7 Fix postgres database for grafana (#40656)
add bd102581f7f Run portable ValidatesRunner tests one at a time (#40632)
add 62357960fc3 Remove a race with another file write that is not
essential to test presence of duplicates. (#40596)
add a8ee2b84d37 [Spark 4] Group the streaming stateful ParDo by encoded
key bytes to support null keys (#40516)
add 59a8647c15a Update nlp_tensorflow_streaming.ipynb (#40641)
add 5dc12703dc9 [Prism] Keep a stateful key in progress while a split
primary holds it (#40624)
add 05c48794a86 allow dropping pk or watermark column (#40457)
add 3bef0c0cb08 [Python] Fail bundles on state write errors (#39992)
add 3ab18d061df [Go SDK] Add portable logical type registration and
standard date and micros_instant types (#40154)
add f8b1e8cdd75 [Python] Add typed decision questions and model adapters
(#40220)
add 75d7e2c467f Process deletes before adds when committing OrderedList
state updates. Otherwise we delete the newly added id from the idTracker.
(#40659)
add b4ecb26ada0 Modernize legacy DoFn in BatchElementsExample (#40557)
(#40560)
add 18db4896cf1 [Multikey] [Dataflow Streaming] Replace KeyGroupWorkQueue
with a sharded KeyGroupIndex (#40593)
add c61c5cdb70e Merge branch 'master' of
https://github.com/ahmedabu98/beam into sql-shell-cancel
No new revisions were added by this update.
Summary of changes:
.../beam_PostCommit_Java_PVR_Flink_Batch.json | 2 +-
.../beam_PostCommit_Java_PVR_Flink_Streaming.json | 2 +-
...eam_PostCommit_Java_ValidatesRunner_Spark4.json | 2 +-
...a_ValidatesRunner_SparkStructuredStreaming.json | 2 +-
.../beam_PostCommit_Python_Versions.json | 2 +-
.../beam_PostCommit_Yaml_Xlang_Direct.json | 2 +-
...ow.json => beam_PreCommit_Java_IOs_Direct.json} | 0
.../beam_PreCommit_Yaml_Xlang_Direct.json | 2 +-
.github/workflows/beam_PreCommit_GHA.yml | 2 +-
.github/workflows/codeql.yml | 4 +-
.../provisioning/datasources/beampostgresql.yaml | 2 +-
.test-infra/tools/README.md | 18 +
.test-infra/tools/build.gradle | 10 +
.test-infra/tools/stale_gcs_buckets_cleaner.py | 112 +++++
.../tools/test_stale_gcs_buckets_cleaner.py | 202 ++++++++
CHANGES.md | 7 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 4 +-
.../apache/beam/examples/BatchElementsExample.java | 7 +-
.../datatokenization/utils/SchemasUtils.java | 9 +-
.../subprocess/utils/CallingSubProcessUtils.java | 9 +-
.../examples/subprocess/utils/ExecutableFile.java | 9 +-
.../beam-ml/nlp_tensorflow_streaming.ipynb | 6 +-
infra/iam/users.yml | 7 +-
.../it/clickhouse/ClickHouseResourceManager.java | 11 +-
.../it/gcp/bigtable/BigtableResourceManager.java | 9 +-
.../beam/it/splunk/SplunkResourceManager.java | 9 +-
.../frontend/lib/firebase_options.dart | 2 +-
.../construction/SerializablePipelineOptions.java | 9 +-
.../beam/runners/core/metrics/BoundedTrieData.java | 16 +-
.../FlinkStreamingPortablePipelineTranslator.java | 9 +-
.../FlinkExecutableStageContextFactory.java | 51 +-
.../streaming/io/UnboundedSourceWrapper.java | 10 +-
.../streaming/state/FlinkStateInternals.java | 9 +-
runners/flink/job-server/flink_job_server.gradle | 2 +
.../FlinkStreamingPortablePipelineTranslator.java | 9 +-
.../FlinkExecutableStageContextFactory.java | 51 +-
.../streaming/io/UnboundedSourceWrapper.java | 10 +-
.../io/source/FlinkSourceSplitEnumerator.java | 9 +-
.../io/source/LazyFlinkSourceSplitEnumerator.java | 9 +-
.../streaming/stableinput/BufferingDoFnRunner.java | 9 +-
.../stableinput/KeyedBufferingElementsHandler.java | 9 +-
.../streaming/state/FlinkStateInternals.java | 9 +-
.../dataflow/worker/StreamingDataflowWorker.java | 7 +-
.../runners/dataflow/worker/streaming/Work.java | 3 +-
.../dataflow/worker/util/BoundedQueueExecutor.java | 224 +++++++--
.../dataflow/worker/util/KeyGroupWorkIndex.java | 190 ++++++++
.../dataflow/worker/util/KeyGroupWorkQueue.java | 472 -------------------
.../worker/windmill/state/WindmillOrderedList.java | 28 +-
.../worker/StreamingDataflowWorkerTest.java | 12 +-
.../worker/util/BoundedQueueExecutorTest.java | 428 +++++++++++------
.../worker/util/KeyGroupWorkIndexTest.java | 319 +++++++++++++
.../worker/util/KeyGroupWorkQueueTest.java | 523 ---------------------
.../windmill/state/WindmillStateInternalsTest.java | 105 +++++
.../processing/StreamingCommitFinalizerTest.java | 2 +-
.../failures/WorkFailureProcessorTest.java | 2 +-
.../work/refresh/ActiveWorkRefresherTest.java | 2 +-
.../control/DefaultJobBundleFactory.java | 9 +-
.../org/apache/beam/runners/jet/JetRunner.java | 9 +-
.../beam/runners/jet/processors/AssignWindowP.java | 9 +-
.../io/streaming/BeamReaderCache.java | 2 +-
.../io/streaming/UnboundedSourceDataset.java | 2 +-
.../batch/GroupByKeyTranslatorBatch.java | 2 +-
.../translation/helpers/EncoderHelpers.java | 1 +
.../streaming/ReadUnboundedTranslator.java | 2 +-
.../StatefulParDoStreamingTranslator.java | 10 +-
.../streaming/state/BeamStatefulProcessor.java | 13 +-
.../io/streaming/BeamMicroBatchSourceTest.java | 12 +-
.../streaming/StatefulParDoStreamingTest.java | 36 ++
.../spark/metrics/SparkBeamMetricSource.java | 9 +-
.../beam/runners/spark/metrics/sink/CsvSink.java | 9 +-
.../runners/spark/metrics/sink/GraphiteSink.java | 9 +-
.../metrics/SparkBeamMetricSource.java | 9 +-
.../metrics/sink/CodahaleCsvSink.java | 9 +-
.../metrics/sink/CodahaleGraphiteSink.java | 9 +-
.../batch/GroupByKeyTranslatorBatch.java | 2 +-
.../batch/functions/SideInputValues.java | 2 +-
.../translation/helpers/CoderHelpers.java | 59 ---
.../translation/helpers/EncoderHelpers.java | 1 +
.../SparkBatchPortablePipelineTranslator.java | 9 +-
.../SparkDatasetPortablePipelineTranslator.java | 9 +-
.../SparkStreamingPortablePipelineTranslator.java | 9 +-
sdks/go.mod | 8 +-
sdks/go.sum | 16 +-
sdks/go/pkg/beam/core/graph/coder/row.go | 16 +
sdks/go/pkg/beam/core/graph/coder/row_decoder.go | 37 ++
sdks/go/pkg/beam/core/graph/coder/row_encoder.go | 21 +
sdks/go/pkg/beam/core/graph/coder/row_test.go | 105 +++++
.../core/runtime/graphx/schema/logicaltypes.go | 142 +++++-
.../beam/core/runtime/graphx/schema/schema_test.go | 30 ++
.../graphx/schema/standard_logical_types.go | 156 ++++++
.../graphx/schema/standard_logical_types_test.go | 421 +++++++++++++++++
.../pkg/beam/core/runtime/graphx/serialize_test.go | 16 +
sdks/go/pkg/beam/io/bigqueryio/bigquery.go | 54 ++-
sdks/go/pkg/beam/io/bigqueryio/bigquery_test.go | 125 +++++
.../pkg/beam/runners/prism/internal/engine/data.go | 7 +-
.../runners/prism/internal/engine/data_test.go | 37 ++
.../prism/internal/engine/elementmanager.go | 15 +-
.../internal/engine/elementmanager_split_test.go | 194 ++++++++
.../pkg/beam/runners/prism/internal/preprocess.go | 26 +-
.../beam/runners/prism/internal/preprocess_test.go | 65 +++
sdks/go/test/integration/primitives/pardo.go | 15 +
sdks/go/test/integration/primitives/pardo_test.go | 5 +
.../go/test/regression/coders/fromyaml/fromyaml.go | 210 ++++++---
.../org/apache/beam/sdk/transforms/Sample.java | 9 +-
.../beam/fn/harness/ExternalWorkerService.java | 33 +-
.../java/org/apache/beam/fn/harness/FnHarness.java | 1 +
.../fn/harness/control/ProcessBundleHandler.java | 3 +
.../beam/fn/harness/data/BeamFnDataGrpcClient.java | 11 +-
.../harness/state/BeamFnStateGrpcClientCache.java | 20 +-
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 242 +++++++++-
.../beam/sdk/io/gcp/bigquery/BigQueryOptions.java | 10 +
.../sdk/io/gcp/bigquery/BigQueryServicesImpl.java | 23 +-
.../gcp/bigquery/BigQueryStorageStreamSource.java | 21 +-
.../apache/beam/sdk/io/gcp/spanner/SpannerIO.java | 4 +-
.../io/gcp/bigquery/BigQueryServicesImplTest.java | 54 +++
.../io/gcp/bigquery/CleanupTempTableDoFnTest.java | 153 ++++++
.../gcp/spanner/SpannerIOReadChangeStreamTest.java | 52 ++
.../beam/sdk/io/iceberg/IcebergScanConfig.java | 78 ++-
.../io/iceberg/cdc/IncrementalChangelogSource.java | 34 +-
.../IcebergCdcReadSchemaTransformProviderTest.java | 7 +-
.../beam/sdk/io/iceberg/IcebergIOReadTest.java | 18 -
.../beam/sdk/io/iceberg/IcebergScanConfigTest.java | 77 ++-
.../cdc/IncrementalChangelogSourceTest.java | 87 +++-
.../sdk/ml/inference/remote/RemoteInference.java | 4 +-
.../ml/inference/remote/RemoteInferenceTest.java | 3 -
sdks/python/apache_beam/dataframe/io_it_test.py | 7 +-
sdks/python/apache_beam/dataframe/io_test.py | 11 +-
.../apache_beam/examples/inference/README.md | 8 +-
.../examples/inference/decision_models/README.md | 304 ++++++++++++
.../inference/decision_models/__init__.py} | 3 +-
.../inference/decision_models/fraud_review.py | 126 +++++
.../inference/decision_models/fraud_review_test.py | 86 ++++
.../inference/decision_models/local_model.py | 78 +++
.../inference/decision_models/message_router.py | 199 ++++++++
.../inference/decision_models/message_router.svg | 98 ++++
.../decision_models/message_router_test.py | 75 +++
.../inference/decision_models}/requirements.txt | 3 +-
.../secret_manager}/__init__.py | 2 +-
.../aws_credentials_requirements.txt | 8 +-
.../secret_manager/aws_credentials_s3_pipeline.py | 75 +++
.../secret_manager/aws_credentials_s3_read.py | 60 +++
.../secret_manager/aws_credentials_secret_read.py | 87 ++++
.../apache_beam/examples/snippets/snippets_test.py | 8 +-
.../io/components/adaptive_throttler.py | 4 +-
sdks/python/apache_beam/io/concat_source.py | 4 +-
sdks/python/apache_beam/io/concat_source_test.py | 9 +
sdks/python/apache_beam/io/fileio.py | 3 +-
sdks/python/apache_beam/io/fileio_test.py | 10 +-
sdks/python/apache_beam/io/gcp/bigquery.py | 121 +++--
.../apache_beam/io/gcp/bigquery_change_history.py | 42 +-
.../io/gcp/bigquery_change_history_test.py | 75 +++
.../apache_beam/io/gcp/bigquery_file_loads.py | 46 +-
.../apache_beam/io/gcp/bigquery_file_loads_test.py | 47 ++
.../apache_beam/io/gcp/bigquery_read_internal.py | 11 +-
sdks/python/apache_beam/io/gcp/bigquery_test.py | 355 ++++++++++++++
sdks/python/apache_beam/io/gcp/bigquery_tools.py | 162 ++++++-
.../apache_beam/io/gcp/bigquery_tools_test.py | 232 ++++++++-
.../apache_beam/io/requestresponse_it_test.py | 5 +
sdks/python/apache_beam/ml/inference/base.py | 33 +-
sdks/python/apache_beam/ml/inference/base_test.py | 35 ++
sdks/python/apache_beam/ml/inference/decision.py | 246 ++++++++++
.../apache_beam/ml/inference/decision_test.py | 254 ++++++++++
.../apache_beam/ml/inference/model_manager.py | 16 +-
.../apache_beam/ml/inference/model_manager_test.py | 54 +++
.../apache_beam/ml/inference/typesafe_inference.py | 164 +++++++
.../ml/inference/typesafe_inference_it_test.py | 93 ++++
.../ml/inference/typesafe_inference_test.py | 256 ++++++++++
.../ml/inference/typesafe_tests_requirements.txt} | 3 +-
sdks/python/apache_beam/ml/inference/utils.py | 13 +-
.../python/apache_beam/options/pipeline_options.py | 11 +
.../apache_beam/runners/worker/bundle_processor.py | 35 +-
.../runners/worker/bundle_processor_test.py | 111 +++++
sdks/python/apache_beam/testing/test_utils.py | 25 +
sdks/python/apache_beam/testing/test_utils_test.py | 37 ++
.../enrichment_handlers/cloudsql_it_test.py | 3 +
.../apache_beam/transforms/periodicsequence.py | 5 +-
sdks/python/apache_beam/utils/retry.py | 2 +-
sdks/python/apache_beam/utils/retry_test.py | 18 +
.../e2e/gcs_avro_to_cloud_bigtable.yaml | 10 +-
sdks/python/conftest.py | 38 +-
sdks/python/setup.py | 3 +-
.../src/apache_beam/internal/serialize.ts | 2 +-
sdks/typescript/src/apache_beam/serialization.ts | 17 +-
sdks/typescript/test/primitives_test.ts | 6 +-
184 files changed, 7783 insertions(+), 1979 deletions(-)
copy .github/trigger_files/{IO_Iceberg_Integration_Tests_Dataflow.json =>
beam_PreCommit_Java_IOs_Direct.json} (100%)
create mode 100644 .test-infra/tools/stale_gcs_buckets_cleaner.py
create mode 100644 .test-infra/tools/test_stale_gcs_buckets_cleaner.py
create mode 100644
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/KeyGroupWorkIndex.java
delete mode 100644
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/KeyGroupWorkQueue.java
create mode 100644
runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/util/KeyGroupWorkIndexTest.java
delete mode 100644
runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/util/KeyGroupWorkQueueTest.java
delete mode 100644
runners/spark/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/CoderHelpers.java
create mode 100644
sdks/go/pkg/beam/core/runtime/graphx/schema/standard_logical_types.go
create mode 100644
sdks/go/pkg/beam/core/runtime/graphx/schema/standard_logical_types_test.go
create mode 100644
sdks/go/pkg/beam/runners/prism/internal/engine/elementmanager_split_test.go
create mode 100644
sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/CleanupTempTableDoFnTest.java
create mode 100644
sdks/python/apache_beam/examples/inference/decision_models/README.md
copy sdks/python/{.coveragerc =>
apache_beam/examples/inference/decision_models/__init__.py} (95%)
create mode 100644
sdks/python/apache_beam/examples/inference/decision_models/fraud_review.py
create mode 100644
sdks/python/apache_beam/examples/inference/decision_models/fraud_review_test.py
create mode 100644
sdks/python/apache_beam/examples/inference/decision_models/local_model.py
create mode 100644
sdks/python/apache_beam/examples/inference/decision_models/message_router.py
create mode 100644
sdks/python/apache_beam/examples/inference/decision_models/message_router.svg
create mode 100644
sdks/python/apache_beam/examples/inference/decision_models/message_router_test.py
copy sdks/python/apache_beam/{testing/benchmarks/wordcount =>
examples/inference/decision_models}/requirements.txt (86%)
copy sdks/python/apache_beam/{typehints/testing =>
examples/secret_manager}/__init__.py (92%)
copy infra/iam/config.auto.tfvars =>
sdks/python/apache_beam/examples/secret_manager/aws_credentials_requirements.txt
(79%)
create mode 100644
sdks/python/apache_beam/examples/secret_manager/aws_credentials_s3_pipeline.py
create mode 100644
sdks/python/apache_beam/examples/secret_manager/aws_credentials_s3_read.py
create mode 100644
sdks/python/apache_beam/examples/secret_manager/aws_credentials_secret_read.py
create mode 100644 sdks/python/apache_beam/ml/inference/decision.py
create mode 100644 sdks/python/apache_beam/ml/inference/decision_test.py
create mode 100644 sdks/python/apache_beam/ml/inference/typesafe_inference.py
create mode 100644
sdks/python/apache_beam/ml/inference/typesafe_inference_it_test.py
create mode 100644
sdks/python/apache_beam/ml/inference/typesafe_inference_test.py
copy sdks/python/{.coveragerc =>
apache_beam/ml/inference/typesafe_tests_requirements.txt} (96%)
