This is an automated email from the ASF dual-hosted git repository.
yhu pushed a change to branch flink2.0-tag
in repository https://gitbox.apache.org/repos/asf/beam.git
discard 2c397714a19 Exercise Flink 2.0 Python Validates Runner tests
add f96db3733c2 [SQL] Alter Catalog and Table statements (#36571)
add dd0f41d3fe7 Fix _OrderedUnionCoder is_deterministic check. (#37550)
add 25d3a2e46bf Add States for Pause (#37546)
add 257fa5a46d3 Fix Components visibility (#37551)
add 6dcb908e9fe Add back flex API option with improved pydoc (#37491)
add 5142be92511 [Java FnApi] Improve FinalizeCallbackManager to cleanup
timeout queue and use recommended ReentrantLock+Condition (#37304)
add f5aa2af52bd Bump pip from 25.3 to 26.0 in
/sdks/python/container/ml/py313 (#37485)
add 44eb7f63cdc Add RateLimiter Abstractions (#37547)
add d51ea3afb88 Add access to pipeline options during pipeline
construction and translation. (#37522)
add e8920c79a0d Add logic to handle end timestamp of mutable change stream
bounded query (#37459)
add 801bf89e3da Parse proto bytes change stream record in
ChangeStreamRecordMapper (#37427)
add d9adfa9c34f fix unparse bug (#37562)
add 195cc593590 Update python-pipeline-dependencies.md (#37561)
add 7df489cb7c6 add datadog it support (#37517)
add 91d3b6b1d39 move enum to top level class
add 4a2afe85f9f move enum to sdk core changed boolean causedByDrain in
WindowedValue interface to enum. spotless
add aaecfa8f545 Merge pull request #37549: Move CausedByDrain to top
level, move types
add 0228126c1b7 Update reviewers (#37572)
add d6390b86f0f Download performance looks to static while building
website (#37559)
add 22d1c576272 Add OOM handling for RunInference with model manager
(#37557)
add bafb270b48e Update go linter version (#37579)
add cea7923eef6 Use onlyif (#37578)
add 8c9d75567e7 Replace VarInt.getLength with Protobuf's branchless
implementation (#36959)
add 9ea7380de85 Refactor update compatibility checks and pipeline options
access (#37566)
add f29e3d3841c Exercise Flink 2.0 Python Validates Runner tests
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (2c397714a19)
\
N -- N -- N refs/heads/flink2.0-tag (f29e3d3841c)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.github/REVIEWERS.yml | 1 -
.github/trigger_files/beam_PostCommit_SQL.json | 2 +-
.../trigger_files/beam_PreCommit_Python_Dill.json | 2 +-
.github/trigger_files/beam_PreCommit_SQL.json | 2 +-
.../beam_PostCommit_Python_Portable_Flink.yml | 6 +-
.../workflows/beam_PreCommit_Website_Stage_GCS.yml | 12 +-
.github/workflows/beam_Publish_Website.yml | 8 +-
it/{mongodb => datadog}/build.gradle | 40 ++-
.../beam/it/datadog/DatadogClientFactory.java} | 28 +-
.../apache/beam/it/datadog/DatadogLogEntry.java | 31 +-
.../beam/it/datadog/DatadogResourceManager.java | 335 +++++++++++++++++++++
.../datadog/DatadogResourceManagerException.java} | 10 +-
.../it/datadog/DatadogResourceManagerUtils.java | 70 +++++
.../datadog/conditions/DatadogLogEntriesCheck.java | 91 ++++++
.../beam/it/datadog/conditions}/package-info.java | 4 +-
.../beam/it/datadog/matchers/DatadogAsserts.java} | 36 ++-
.../beam/it/datadog/matchers}/package-info.java | 8 +-
.../org/apache/beam/it/datadog}/package-info.java | 8 +-
.../beam/it/datadog/DatadogResourceManagerIT.java | 77 +++++
.../it/datadog/DatadogResourceManagerTest.java | 188 ++++++++++++
.../datadog/DatadogResourceManagerUtilsTest.java | 67 +++++
playground/backend/cmd/server/controller.go | 2 +-
playground/backend/cmd/server/server.go | 4 +-
playground/backend/env_setup.sh | 2 +-
.../beam/runners/core/ReduceFnContextFactory.java | 10 +-
.../core/SplittableParDoViaKeyedWorkItems.java | 3 +-
.../apache/beam/runners/core/TimerInternals.java | 12 +-
.../runners/core/InMemoryTimerInternalsTest.java | 37 +--
.../beam/runners/core/KeyedWorkItemCoderTest.java | 3 +-
.../apache/beam/runners/core/ReduceFnTester.java | 5 +-
.../beam/runners/core/SimpleDoFnRunnerTest.java | 3 +-
.../SimplePushbackSideInputDoFnRunnerTest.java | 5 +-
.../beam/runners/core/TimerInternalsTest.java | 59 +---
.../core/triggers/TriggerStateMachineTester.java | 10 +-
.../runners/direct/DirectTimerInternalsTest.java | 13 +-
...cycleManagerRemovingTransformEvaluatorTest.java | 3 +-
.../beam/runners/direct/EvaluationContextTest.java | 3 +-
.../beam/runners/direct/WatermarkManagerTest.java | 85 ++----
.../FlinkStreamingPortablePipelineTranslator.java | 21 +-
.../FlinkStreamingPortablePipelineTranslator.java | 21 +-
.../beam/runners/dataflow/BatchViewOverrides.java | 5 +-
.../beam/runners/dataflow/util/MonitoringUtil.java | 2 +
.../runners/dataflow/util/MonitoringUtilTest.java | 2 +
.../dataflow/worker/UngroupedWindmillReader.java | 7 +-
.../dataflow/worker/WindmillKeyedWorkItem.java | 7 +-
.../dataflow/worker/WindmillTimerInternals.java | 5 +-
.../dataflow/worker/util/ValueInEmptyWindows.java | 5 +-
.../worker/StreamingGroupAlsoByWindowFnsTest.java | 3 +-
...eamingKeyedWorkItemSideInputDoFnRunnerTest.java | 3 +-
.../worker/StreamingModeExecutionContextTest.java | 3 +-
.../worker/StreamingSideInputFetcherTest.java | 3 +-
.../dataflow/worker/UserParDoFnFactoryTest.java | 7 +-
.../dataflow/worker/WindmillKeyedWorkItemTest.java | 9 +-
.../windmill/state/WindmillTagEncodingV1Test.java | 15 +-
.../windmill/state/WindmillTagEncodingV2Test.java | 3 +-
.../samza/runtime/ClassicBundleManager.java | 3 +-
.../samza/runtime/PortableBundleManager.java | 3 +-
.../runners/samza/runtime/KeyedTimerDataTest.java | 3 +-
.../runtime/SamzaTimerInternalsFactoryTest.java | 25 +-
.../apache/beam/runners/spark/util/TimerUtils.java | 5 +-
.../main/java/org/apache/beam/sdk/util/VarInt.java | 10 +-
.../org/apache/beam/sdk/values/CausedByDrain.java | 6 +-
.../org/apache/beam/sdk/values/OutputBuilder.java | 2 +-
.../org/apache/beam/sdk/values/WindowedValue.java | 2 +-
.../org/apache/beam/sdk/values/WindowedValues.java | 58 ++--
.../apache/beam/sdk/util/WindowedValueTest.java | 5 +-
.../extensions/sql/expansion-service/build.gradle | 1 -
sdks/java/extensions/sql/iceberg/build.gradle | 2 +
.../provider/iceberg/IcebergAlterTableOps.java | 50 +++
.../sql/meta/provider/iceberg/IcebergCatalog.java | 50 +--
.../meta/provider/iceberg/IcebergMetastore.java | 16 +-
.../iceberg/BeamSqlCliIcebergAlterTest.java | 231 ++++++++++++++
.../extensions/sql/src/main/codegen/config.fmpp | 7 +
.../sql/src/main/codegen/includes/parserImpls.ftl | 120 +++++++-
.../sdk/extensions/sql/impl/CatalogSchema.java | 4 +
...{SqlCreateCatalog.java => SqlAlterCatalog.java} | 164 +++++-----
.../extensions/sql/impl/parser/SqlAlterTable.java | 254 ++++++++++++++++
.../sql/impl/parser/SqlCreateExternalTable.java | 2 +-
.../extensions/sql/impl/parser/SqlDdlNodes.java | 41 +++
.../sql/impl/parser/SqlDropDatabase.java | 6 +-
.../extensions/sql/impl/parser/SqlDropTable.java | 6 +-
.../extensions/sql/impl/parser/SqlUseDatabase.java | 2 +-
.../sdk/extensions/sql/meta/catalog/Catalog.java | 3 +
.../sql/meta/catalog/InMemoryCatalog.java | 8 +-
.../sql/meta/provider/AlterTableOps.java | 45 +++
.../sql/meta/provider/TableProvider.java | 5 +
.../sql/meta/provider/test/AlterTestTableOps.java | 101 +++++++
.../sql/meta/provider/test/TestTableProvider.java | 21 ++
.../sql/meta/store/InMemoryMetaStore.java | 11 +
.../sdk/extensions/sql/BeamSqlCliCatalogTest.java | 93 ++++++
.../beam/sdk/extensions/sql/BeamSqlCliTest.java | 249 +++++++++++++++
.../fn/harness/control/FinalizeBundleHandler.java | 182 ++++++-----
.../harness/control/FinalizeBundleHandlerTest.java | 49 +++
.../components/deadletterqueue}/package-info.java | 4 +-
.../deadletterqueue/sinks}/package-info.java | 4 +-
.../io/components/ratelimiter/RateLimiter.java} | 25 +-
.../ratelimiter/RateLimiterContext.java} | 14 +-
.../components/ratelimiter/RateLimiterFactory.java | 57 ++++
.../io/components/ratelimiter}/package-info.java | 4 +-
.../io/components/throttling}/package-info.java | 4 +-
.../apache/beam/sdk/io/gcp/spanner/SpannerIO.java | 6 +-
.../changestreams/action/ActionFactory.java | 6 +-
.../action/QueryChangeStreamAction.java | 33 +-
.../changestreams/dao/ChangeStreamResultSet.java | 31 ++
.../gcp/spanner/changestreams/dao/DaoFactory.java | 4 +
.../dofn/ReadChangeStreamPartitionDoFn.java | 5 +-
.../mapper/ChangeStreamRecordMapper.java | 13 +-
.../action/QueryChangeStreamActionTest.java | 120 +++++++-
.../dao/ChangeStreamResultSetTest.java | 60 ++++
.../dofn/ReadChangeStreamPartitionDoFnTest.java | 25 +-
...annerChangeStreamPlacementTablePostgresIT.java} | 4 +-
.../mapper/ChangeStreamRecordMapperTest.java | 105 +++++++
.../beam/sdk/io/iceberg/IcebergCatalogConfig.java | 103 ++++++-
.../apache/beam/sdk/io/iceberg/PartitionUtils.java | 37 +++
.../beam/sdk/io/iceberg/FilterUtilsTest.java | 6 -
sdks/python/apache_beam/coders/coders.py | 34 +--
.../apache_beam/coders/coders_test_common.py | 208 +++++++------
sdks/python/apache_beam/coders/typecoders.py | 1 -
.../apache_beam/io/gcp/bigquery_file_loads.py | 4 +-
.../apache_beam/io/gcp/bigquery_file_loads_test.py | 15 +-
sdks/python/apache_beam/ml/inference/base.py | 39 ++-
sdks/python/apache_beam/ml/inference/base_test.py | 26 ++
.../apache_beam/ml/inference/gemini_inference.py | 26 +-
.../ml/inference/gemini_inference_test.py | 25 ++
.../apache_beam/ml/inference/model_manager.py | 63 ++--
.../python/apache_beam/options/pipeline_options.py | 19 ++
.../options/pipeline_options_context.py | 65 ++++
.../options/pipeline_options_context_test.py | 335 +++++++++++++++++++++
.../apache_beam/options/pipeline_options_test.py | 66 ++++
sdks/python/apache_beam/pipeline.py | 16 +
sdks/python/apache_beam/pipeline_test.py | 2 +
.../runners/portability/flink_runner_test.py | 13 +-
sdks/python/apache_beam/transforms/external.py | 7 +-
sdks/python/apache_beam/transforms/ptransform.py | 6 +-
sdks/python/apache_beam/transforms/util.py | 36 +--
sdks/python/apache_beam/transforms/util_test.py | 78 +----
.../container/ml/py313/base_image_requirements.txt | 2 +-
settings.gradle.kts | 2 +
website/Dockerfile | 17 ++
website/build.gradle | 55 ++++
.../sdks/python-pipeline-dependencies.md | 14 +
website/www/site/data/performance.yaml | 1 +
.../site/layouts/shortcodes/performance_looks.html | 21 +-
143 files changed, 4229 insertions(+), 888 deletions(-)
copy it/{mongodb => datadog}/build.gradle (53%)
copy
it/{splunk/src/main/java/org/apache/beam/it/splunk/SplunkClientFactory.java =>
datadog/src/main/java/org/apache/beam/it/datadog/DatadogClientFactory.java}
(60%)
copy
sdks/java/io/datadog/src/main/java/org/apache/beam/sdk/io/datadog/DatadogEvent.java
=> it/datadog/src/main/java/org/apache/beam/it/datadog/DatadogLogEntry.java
(78%)
create mode 100644
it/datadog/src/main/java/org/apache/beam/it/datadog/DatadogResourceManager.java
copy
it/{mongodb/src/main/java/org/apache/beam/it/mongodb/MongoDBResourceManagerException.java
=>
datadog/src/main/java/org/apache/beam/it/datadog/DatadogResourceManagerException.java}
(75%)
create mode 100644
it/datadog/src/main/java/org/apache/beam/it/datadog/DatadogResourceManagerUtils.java
create mode 100644
it/datadog/src/main/java/org/apache/beam/it/datadog/conditions/DatadogLogEntriesCheck.java
copy it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp/pubsub =>
datadog/src/main/java/org/apache/beam/it/datadog/conditions}/package-info.java
(88%)
copy
it/{mongodb/src/main/java/org/apache/beam/it/mongodb/matchers/MongoDBAsserts.java
=>
datadog/src/main/java/org/apache/beam/it/datadog/matchers/DatadogAsserts.java}
(56%)
copy
{examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization
=>
it/datadog/src/main/java/org/apache/beam/it/datadog/matchers}/package-info.java
(70%)
copy
{examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization
=> it/datadog/src/main/java/org/apache/beam/it/datadog}/package-info.java (70%)
create mode 100644
it/datadog/src/test/java/org/apache/beam/it/datadog/DatadogResourceManagerIT.java
create mode 100644
it/datadog/src/test/java/org/apache/beam/it/datadog/DatadogResourceManagerTest.java
create mode 100644
it/datadog/src/test/java/org/apache/beam/it/datadog/DatadogResourceManagerUtilsTest.java
copy learning/tour-of-beam/frontend/lib/constants/storage_keys.dart =>
sdks/java/core/src/main/java/org/apache/beam/sdk/values/CausedByDrain.java (89%)
create mode 100644
sdks/java/extensions/sql/iceberg/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/iceberg/IcebergAlterTableOps.java
create mode 100644
sdks/java/extensions/sql/iceberg/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/iceberg/BeamSqlCliIcebergAlterTest.java
copy
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/{SqlCreateCatalog.java
=> SqlAlterCatalog.java} (54%)
create mode 100644
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/SqlAlterTable.java
create mode 100644
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/AlterTableOps.java
create mode 100644
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/test/AlterTestTableOps.java
copy {it/splunk/src/main/java/org/apache/beam/it/splunk =>
sdks/java/io/components/src/main/java/org/apache/beam/sdk/io/components/deadletterqueue}/package-info.java
(86%)
copy {it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/dataflow
=>
sdks/java/io/components/src/main/java/org/apache/beam/sdk/io/components/deadletterqueue/sinks}/package-info.java
(86%)
copy sdks/java/{core/src/main/java/org/apache/beam/sdk/util/ShardedFile.java
=>
io/components/src/main/java/org/apache/beam/sdk/io/components/ratelimiter/RateLimiter.java}
(50%)
copy sdks/java/{core/src/main/java/org/apache/beam/sdk/metrics/Metric.java =>
io/components/src/main/java/org/apache/beam/sdk/io/components/ratelimiter/RateLimiterContext.java}
(77%)
create mode 100644
sdks/java/io/components/src/main/java/org/apache/beam/sdk/io/components/ratelimiter/RateLimiterFactory.java
copy {it/kafka/src/main/java/org/apache/beam/it/kafka =>
sdks/java/io/components/src/main/java/org/apache/beam/sdk/io/components/ratelimiter}/package-info.java
(87%)
copy
{examples/java/cdap/hubspot/src/main/java/org/apache/beam/examples/complete/cdap/hubspot/options
=>
sdks/java/io/components/src/main/java/org/apache/beam/sdk/io/components/throttling}/package-info.java
(87%)
create mode 100644
sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/dao/ChangeStreamResultSetTest.java
copy
sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/it/{SpannerChangeStreamPostgresIT.java
=> SpannerChangeStreamPlacementTablePostgresIT.java} (98%)
create mode 100644 sdks/python/apache_beam/options/pipeline_options_context.py
create mode 100644
sdks/python/apache_beam/options/pipeline_options_context_test.py
