This is an automated email from the ASF dual-hosted git repository.
davidzollo pushed a change to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
from 6bb02da900 [Improve][Connector-V2][Iceberg] Align OptionRule with
docs and strengthen factory test (#11675)
add 3488b1caf5 [Feature][Core] Error Data Bypass Capability for Transform
& Sink (Dead Letter Queue) (#10306)
No new revisions were added by this update.
Summary of changes:
.../introduction/configuration/error-handling.md | 363 ++++++++
docs/sidebars.js | 1 +
docs/zh/engines/zeta/realtime-observability.md | 6 +-
docs/zh/engines/zeta/rest-api-job-lifecycle.md | 2 +-
.../introduction/configuration/error-handling.md | 363 ++++++++
seatunnel-api/pom.xml | 5 +
.../api/common/error/RowErrorClassification.java | 13 +-
.../api/common/error/RowErrorCollector.java | 16 +-
.../seatunnel/api/common/error/RowErrorEvent.java | 63 ++
.../error/RowErrorHandlingFatalException.java | 9 +-
.../error/RowErrorPhase.java} | 12 +-
.../error/SupportRowLevelErrorClassifier.java} | 35 +-
.../seatunnel/api/common/metrics/MetricNames.java | 6 +
.../api/sink/DefaultSinkWriterContext.java | 11 +
.../org/apache/seatunnel/api/sink/SinkWriter.java | 25 +
.../multitablesink/MultiTableRowErrorHandler.java | 58 ++
.../api/sink/multitablesink/MultiTableSink.java | 34 +-
.../sink/multitablesink/MultiTableSinkWriter.java | 108 ++-
.../api/sink/multitablesink/MultiTableState.java | 14 +
.../multitablesink/MultiTableWriterRunnable.java | 151 +++-
.../api/sink/multitablesink/SinkContextProxy.java | 17 +
.../error/SupportRowLevelErrorClassifierTest.java | 65 ++
.../multitablesink/MultiTableSinkWriterTest.java | 510 ++++++++++-
.../seatunnel/jdbc/internal/JdbcOutputFormat.java | 49 ++
.../BufferReducedBatchStatementExecutor.java | 7 +
.../executor/BufferedBatchStatementExecutor.java | 6 +
.../InsertOrUpdateBatchStatementExecutor.java | 12 +
.../executor/JdbcBatchStatementExecutor.java | 8 +
.../executor/SimpleBatchStatementExecutor.java | 7 +
.../seatunnel/jdbc/sink/JdbcSinkWriter.java | 362 +++++++-
.../jdbc/internal/JdbcOutputFormatTest.java | 183 ++++
.../BufferedBatchStatementExecutorTest.java | 56 ++
.../InsertOrUpdateBatchStatementExecutorTest.java | 154 ++++
.../seatunnel/jdbc/sink/JdbcSinkWriterTest.java | 322 ++++++-
.../container/seatunnel/SeaTunnelContainer.java | 4 +
.../seatunnel/e2e/common/util/ContainerUtil.java | 31 +-
.../connector-seatunnel-e2e-base/pom.xml | 18 +
.../seatunnel/engine/e2e/SinkErrorToMysqlIT.java | 716 +++++++++++++++
.../engine/e2e/TransformErrorToMysqlIT.java | 598 +++++++++++++
.../sink_error_handler_log_max_error_records.conf | 82 ++
.../sink_error_handler_log_mode.conf | 84 ++
.../sink_error_handler_max_error_ratio.conf | 246 ++++++
.../sink_error_handler_max_error_records.conf | 98 +++
...nk_error_handler_no_original_no_stacktrace.conf | 97 +++
.../sink_error_handler_original_only.conf | 97 +++
.../sink_error_handler_queue_block.conf | 115 +++
.../sink_error_handler_queue_drop.conf | 115 +++
.../sink_error_handler_queue_fail.conf | 114 +++
.../sink_error_handler_runtime_bad_error_sink.conf | 102 +++
.../sink_error_handler_system_error.conf | 109 +++
...nk_fakesource_to_mysql_with_bad_error_sink.conf | 99 +++
...ink_fakesource_to_mysql_with_error_handler.conf | 100 +++
...ce_to_mysql_with_error_handler_batch_close.conf | 103 +++
..._with_error_handler_batch_close_constraint.conf | 101 +++
...esource_to_mysql_with_global_error_handler.conf | 99 +++
...fakesource_to_mysql_without_error_handler.conf} | 51 +-
.../transform_and_sink_error_handlers.conf | 148 ++++
...nsform_error_handler_log_max_error_records.conf | 90 ++
.../transform_error_handler_log_mode.conf | 92 ++
.../transform_error_handler_max_error_ratio.conf | 254 ++++++
.../transform_error_handler_max_error_records.conf | 106 +++
...rm_error_handler_no_original_no_stacktrace.conf | 105 +++
.../transform_error_handler_original_only.conf | 105 +++
.../transform_error_handler_queue_block.conf | 123 +++
.../transform_error_handler_queue_drop.conf | 123 +++
.../transform_error_handler_queue_fail.conf | 122 +++
...rm_fakesource_to_mysql_with_bad_error_sink.conf | 107 +++
...orm_fakesource_to_mysql_with_error_handler.conf | 113 +++
...esource_to_mysql_with_global_error_handler.conf | 102 +++
.../transform_no_error_with_error_handler.conf | 107 +++
.../src/test/resources/multi_table_merge.conf | 103 +++
.../batch_fake_to_console_with_transform_name.conf | 1 -
.../engine/server/TaskExecutionService.java | 4 +
.../statestore/AuthoritativeStateStores.java | 10 +
.../DefaultAuthoritativeStateStores.java | 12 +-
.../common/statestore/EngineStateStoreNames.java | 1 +
.../common/statestore/EngineStateStores.java | 9 +
.../statestore/counter/CounterStateStore.java | 10 +
.../hazelcast/HazelcastCounterStateStore.java | 5 +
.../hazelcast/HazelcastEngineStateStores.java | 7 +-
.../server/dag/physical/PhysicalPlanGenerator.java | 5 +-
.../server/execution/TaskExecutionContext.java | 10 +
.../engine/server/task/SeaTunnelTask.java | 34 +-
.../engine/server/task/SourceSeaTunnelTask.java | 2 +-
.../engine/server/task/TransformSeaTunnelTask.java | 10 +-
.../server/task/context/SinkWriterContext.java | 29 +
.../server/task/error/DefaultErrorSinkWriter.java | 967 +++++++++++++++++++++
.../task/error/DefaultRowErrorClassifier.java | 11 +-
.../error/EngineMultiTableRowErrorHandler.java | 178 ++++
.../server/task/error/EngineRowErrorCollector.java | 226 +++++
.../engine/server/task/error/ErrorHandler.java | 292 +++++++
.../server/task/error/ErrorHandlerConfigUtil.java | 262 ++++++
.../server/task/error/ErrorHandlerCounter.java | 55 ++
.../error/ErrorHandlerMode.java} | 40 +-
.../task/error/ErrorHandlingFlatMapTransform.java | 160 ++++
.../task/error/ErrorHandlingMapTransform.java | 159 ++++
.../server/task/error/ErrorHandlingSinkWriter.java | 229 +++++
.../engine/server/task/error/ErrorSinkConfig.java | 28 +-
.../server/task/error/ErrorSinkRowWriter.java | 61 ++
.../task/error/LocalErrorHandlerCounter.java | 33 +-
.../server/task/error/OriginalDataFormat.java | 12 +-
.../error/QueueOverflowPolicy.java} | 41 +-
.../server/task/error/RowErrorClassifier.java | 17 +-
.../engine/server/task/error/RowErrorContext.java | 14 +-
.../engine/server/task/error/StageErrorConfig.java | 33 +-
.../task/error/StateStoreErrorHandlerCounter.java | 115 +++
.../task/error/SynchronizedErrorSinkRowWriter.java | 85 ++
.../engine/server/task/flow/SinkFlowLifeCycle.java | 525 +++++++++--
.../server/task/flow/TransformFlowLifeCycle.java | 166 +++-
.../exports/EngineStateStoreMetricExports.java | 1 +
.../engine/server/trace/StainTraceStage.java | 2 +
.../hazelcast/HazelcastCounterStateStoreTest.java | 13 +
.../DefaultErrorSinkWriterConcurrentTest.java | 467 ++++++++++
.../task/error/DefaultErrorSinkWriterTest.java | 351 ++++++++
.../engine/server/task/error/ErrorHandlerTest.java | 908 +++++++++++++++++++
.../task/error/ErrorHandlingSinkWriterTest.java | 178 ++++
.../flow/SinkFlowLifeCycleErrorOutcomeTest.java | 640 ++++++++++++++
.../exports/EngineStateStoreMetricExportsTest.java | 1 +
.../src/tests/configuration-percent.spec.ts | 1 -
.../src/tests/format-percent.spec.ts | 1 -
.../seatunnel-engine-ui/src/utils/format.ts | 1 -
.../sql/SQLMultiCatalogFlatMapTransform.java | 36 +-
.../seatunnel/transform/sql/SQLTransform.java | 51 +-
.../sql/SQLMultiCatalogFlatMapTransformTest.java | 54 ++
.../seatunnel/transform/sql/SQLTransformTest.java | 40 +
125 files changed, 14300 insertions(+), 335 deletions(-)
create mode 100644 docs/en/introduction/configuration/error-handling.md
create mode 100644 docs/zh/introduction/configuration/error-handling.md
copy
seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/util/JobIdGenerator.java
=>
seatunnel-api/src/main/java/org/apache/seatunnel/api/common/error/RowErrorClassification.java
(75%)
copy
seatunnel-connectors-v2/connector-doris/src/main/java/org/apache/seatunnel/connectors/doris/serialize/DorisSerializer.java
=>
seatunnel-api/src/main/java/org/apache/seatunnel/api/common/error/RowErrorCollector.java
(73%)
create mode 100644
seatunnel-api/src/main/java/org/apache/seatunnel/api/common/error/RowErrorEvent.java
copy
seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/exception/TaskExecuteException.java
=>
seatunnel-api/src/main/java/org/apache/seatunnel/api/common/error/RowErrorHandlingFatalException.java
(72%)
copy
seatunnel-api/src/main/java/org/apache/seatunnel/api/{event/EventListener.java
=> common/error/RowErrorPhase.java} (79%)
copy
seatunnel-api/src/main/java/org/apache/seatunnel/api/{source/Collector.java =>
common/error/SupportRowLevelErrorClassifier.java} (50%)
create mode 100644
seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/multitablesink/MultiTableRowErrorHandler.java
create mode 100644
seatunnel-api/src/test/java/org/apache/seatunnel/api/common/error/SupportRowLevelErrorClassifierTest.java
create mode 100644
seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/JdbcOutputFormatTest.java
create mode 100644
seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/executor/InsertOrUpdateBatchStatementExecutorTest.java
create mode 100644
seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/SinkErrorToMysqlIT.java
create mode 100644
seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/TransformErrorToMysqlIT.java
create mode 100644
seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/error-handling/sink_error_handler_log_max_error_records.conf
create mode 100644
seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/error-handling/sink_error_handler_log_mode.conf
create mode 100644
seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/error-handling/sink_error_handler_max_error_ratio.conf
create mode 100644
seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/error-handling/sink_error_handler_max_error_records.conf
create mode 100644
seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/error-handling/sink_error_handler_no_original_no_stacktrace.conf
create mode 100644
seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/error-handling/sink_error_handler_original_only.conf
create mode 100644
seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/error-handling/sink_error_handler_queue_block.conf
create mode 100644
seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/error-handling/sink_error_handler_queue_drop.conf
create mode 100644
seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/error-handling/sink_error_handler_queue_fail.conf
create mode 100644
seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/error-handling/sink_error_handler_runtime_bad_error_sink.conf
create mode 100644
seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/error-handling/sink_error_handler_system_error.conf
create mode 100644
seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/error-handling/sink_fakesource_to_mysql_with_bad_error_sink.conf
create mode 100644
seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/error-handling/sink_fakesource_to_mysql_with_error_handler.conf
create mode 100644
seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/error-handling/sink_fakesource_to_mysql_with_error_handler_batch_close.conf
create mode 100644
seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/error-handling/sink_fakesource_to_mysql_with_error_handler_batch_close_constraint.conf
create mode 100644
seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/error-handling/sink_fakesource_to_mysql_with_global_error_handler.conf
copy
seatunnel-e2e/{seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_sink_postgres_quick_start.conf
=>
seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/error-handling/sink_fakesource_to_mysql_without_error_handler.conf}
(50%)
create mode 100644
seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/error-handling/transform_and_sink_error_handlers.conf
create mode 100644
seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/error-handling/transform_error_handler_log_max_error_records.conf
create mode 100644
seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/error-handling/transform_error_handler_log_mode.conf
create mode 100644
seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/error-handling/transform_error_handler_max_error_ratio.conf
create mode 100644
seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/error-handling/transform_error_handler_max_error_records.conf
create mode 100644
seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/error-handling/transform_error_handler_no_original_no_stacktrace.conf
create mode 100644
seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/error-handling/transform_error_handler_original_only.conf
create mode 100644
seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/error-handling/transform_error_handler_queue_block.conf
create mode 100644
seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/error-handling/transform_error_handler_queue_drop.conf
create mode 100644
seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/error-handling/transform_error_handler_queue_fail.conf
create mode 100644
seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/error-handling/transform_fakesource_to_mysql_with_bad_error_sink.conf
create mode 100644
seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/error-handling/transform_fakesource_to_mysql_with_error_handler.conf
create mode 100644
seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/error-handling/transform_fakesource_to_mysql_with_global_error_handler.conf
create mode 100644
seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/error-handling/transform_no_error_with_error_handler.conf
create mode 100644
seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/multi_table_merge.conf
create mode 100644
seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/task/error/DefaultErrorSinkWriter.java
copy
seatunnel-api/src/main/java/org/apache/seatunnel/api/event/LoggingEventHandler.java
=>
seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/task/error/DefaultRowErrorClassifier.java
(73%)
create mode 100644
seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/task/error/EngineMultiTableRowErrorHandler.java
create mode 100644
seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/task/error/EngineRowErrorCollector.java
create mode 100644
seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/task/error/ErrorHandler.java
create mode 100644
seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/task/error/ErrorHandlerConfigUtil.java
create mode 100644
seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/task/error/ErrorHandlerCounter.java
copy
seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/{rest/ConfigFormat.java
=> task/error/ErrorHandlerMode.java} (60%)
create mode 100644
seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/task/error/ErrorHandlingFlatMapTransform.java
create mode 100644
seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/task/error/ErrorHandlingMapTransform.java
create mode 100644
seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/task/error/ErrorHandlingSinkWriter.java
copy
seatunnel-edge-agent/seatunnel-edge-agent-connector/src/main/java/org/apache/seatunnel/edge/agent/connector/EdgeSourcePosition.java
=>
seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/task/error/ErrorSinkConfig.java
(67%)
create mode 100644
seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/task/error/ErrorSinkRowWriter.java
copy
seatunnel-api/src/main/java/org/apache/seatunnel/api/table/type/PrimitiveByteArrayType.java
=>
seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/task/error/LocalErrorHandlerCounter.java
(56%)
copy
seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/util/JobIdGenerator.java
=>
seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/task/error/OriginalDataFormat.java
(75%)
copy
seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/{rest/ConfigFormat.java
=> task/error/QueueOverflowPolicy.java} (59%)
copy
seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/SupportSchemaEvolutionSink.java
=>
seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/task/error/RowErrorClassifier.java
(67%)
copy
seatunnel-connectors-v2/connector-typesense/src/main/java/org/apache/seatunnel/connectors/seatunnel/typesense/serialize/source/TypesenseRecord.java
=>
seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/task/error/RowErrorContext.java
(79%)
copy
seatunnel-edge-agent/seatunnel-edge-agent-connector/src/main/java/org/apache/seatunnel/edge/agent/connector/EdgeEvent.java
=>
seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/task/error/StageErrorConfig.java
(55%)
create mode 100644
seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/task/error/StateStoreErrorHandlerCounter.java
create mode 100644
seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/task/error/SynchronizedErrorSinkRowWriter.java
create mode 100644
seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/task/error/DefaultErrorSinkWriterConcurrentTest.java
create mode 100644
seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/task/error/DefaultErrorSinkWriterTest.java
create mode 100644
seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/task/error/ErrorHandlerTest.java
create mode 100644
seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/task/error/ErrorHandlingSinkWriterTest.java
create mode 100644
seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/task/flow/SinkFlowLifeCycleErrorOutcomeTest.java
