This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a change to branch ty/ChangeTsFileVersion
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 5d3d89535fe Try fix CI
add 29529c48947 Subscription: fix unexpected cancellation of workers
during consumer startup & optimize server-side subscription logs & add
synchronized modifier (#13032)
add 89443259f84 Pipe: Fixed the NPE in pipe schema pattern parsing (#13047)
add 57a8823b76b Subscription: support payload size control fallback
strategy & fix issue where subscription events cannot be auto recycled & fix
issue where the reference count of tablet events for tsfile topic cannot
decrease to zero (#13053)
add b4e89452e72 Pipe: Limit the frequency of progress report for
non-forwarding pipe requests to reduce the overhead when sync data between
clusters (#13041)
add cd2d0c50549 Pipe: Enhanced the "isEmpty" judgement for tablets to
prevent NPE (#13051)
add 85ea8d6b262 Fix WALInputStream read ByteBuffer issues (#13059)
add 93b9edcefb0 Enhance remove-datanode function
add 86a191ffc3f fix WALInputStream not closed (#13050)
add 914be1c216b Subscription: improve deduplication logic for
PipeRawTabletInsertionEvent (#13061)
add 6769a7a22eb Fixed concurrency bug in IoTConsensus Region migration
#13070
add c1103a68588 Bump jetty.version from 9.4.53.v20231009 to
9.4.55.v20240627 (#13073)
add ebd31f961c8 Bump org.codehaus.mojo:exec-maven-plugin from 3.1.0 to
3.1.1 (#13075)
add c416846d2c9 Pipe/Subscription: Add 'Create If Not Exists' and 'Drop If
Exists' Support for Pipes, Plugins, and Topics (#12969)
add 7fca497649b Pipe/Subscription: Prevent NPE when some pipe SQL
parameter values are null (#13069)
add 187081e7d5b Bump com.nimbusds:nimbus-jose-jwt from 9.37.2 to 9.37.3
(#13072)
add ca796f2b450 [fix] Change IoTConsensusService and PipeConsensusService
from async to sync (#13077)
add 2ba3dfe9c32 Refactor TsFileValidationTool with abstracting the
sequential scan process (#13066)
add 756b7c2dc52 Improve robustness of removing peer step of region
migration (#13078)
add 10067687131 Pipe: Fix the class not found exception of OPC UA
connector (#13086)
add 330c8b41aa9 Subscription: avoid incomplete resource release of
prefetching queue caused by restarting subscription pipe (#13079)
add ca4a778daa8 Update out-dated profile name in readmes (#13092)
add 05a232fb50e Convert date to LocalDate in sortlist()
add ead49398509 Subscription: java client supports Create Topic If Not
Exists and Drop Topic If Exists (#13081)
add a1a726954d5 PipePlugin/Subscription: The Drop PipePlugin operation
adds a check to see if there is a Topic that uses PipePlugin as a processor
(#13048)
add b05152f4eaf Update multi-language-client CI with path rule (#13095)
add 7b5c07dd035 Fix IoTConsensus log explosion when stoping one node #13101
add e9e0fb1c796 Unify string type inference (#13094)
add ead75f9ad2d Enhance transferLeader retry frequency when migrating
region #13098
add c7059363d01 Add wal compression metric items (#13105)
add c0f82bf89b6 Remove the usage of lt_downsampling_java8 (#13108)
add 69540394cb6 Remove PlainDeviceID
add b17656c3275 Merge remote-tracking branch 'origin/ty/TableModelGrammar'
into ty/ChangeTsFileVersion
No new revisions were added by this update.
Summary of changes:
.github/workflows/multi-language-client.yml | 28 +-
README.md | 4 +-
README_ZH.md | 2 +-
.../IoTDBPipeConditionalOperationsIT.java | 228 +++++++++++
...eSingleEnvDemoIT.java => IoTDBPipeOPCUAIT.java} | 17 +-
.../it/local/IoTDBSubscriptionBasicIT.java | 63 ++++
.../it/local/IoTDBSubscriptionTopicIT.java | 178 +++++++++
.../api/customizer/parameter/PipeParameters.java | 25 +-
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 1 +
.../main/java/org/apache/iotdb/rpc/UrlUtils.java | 9 +-
.../java/org/apache/iotdb/session/Session.java | 9 +-
.../session/subscription/SubscriptionSession.java | 94 ++++-
.../consumer/SubscriptionConsumer.java | 9 +-
.../consumer/SubscriptionPullConsumer.java | 6 +-
.../consumer/SubscriptionPushConsumer.java | 10 +-
.../java/org/apache/iotdb/session/TabletTest.java | 75 +++-
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 14 +-
.../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 9 +
.../iotdb/confignode/manager/ConfigManager.java | 15 +-
.../apache/iotdb/confignode/manager/IManager.java | 11 +-
.../iotdb/confignode/manager/ProcedureManager.java | 13 +-
.../coordinator/plugin/PipePluginCoordinator.java | 12 +-
.../pipe/coordinator/task/PipeTaskCoordinator.java | 11 +-
.../subscription/SubscriptionCoordinator.java | 18 +-
.../persistence/pipe/PipePluginInfo.java | 26 +-
.../confignode/persistence/pipe/PipeTaskInfo.java | 26 +-
.../persistence/subscription/SubscriptionInfo.java | 43 ++-
.../procedure/env/RegionMaintainHandler.java | 10 +-
.../impl/pipe/AbstractOperatePipeProcedureV2.java | 6 +-
.../pipe/plugin/CreatePipePluginProcedure.java | 28 +-
.../impl/pipe/plugin/DropPipePluginProcedure.java | 28 +-
.../runtime/PipeHandleLeaderChangeProcedure.java | 2 +-
.../runtime/PipeHandleMetaChangeProcedure.java | 2 +-
.../impl/pipe/runtime/PipeMetaSyncProcedure.java | 2 +-
.../impl/pipe/task/AlterPipeProcedureV2.java | 6 +-
.../impl/pipe/task/CreatePipeProcedureV2.java | 15 +-
.../impl/pipe/task/DropPipeProcedureV2.java | 2 +-
.../impl/pipe/task/StartPipeProcedureV2.java | 4 +-
.../impl/pipe/task/StopPipeProcedureV2.java | 2 +-
.../AbstractOperateSubscriptionProcedure.java | 15 +-
.../consumer/AlterConsumerGroupProcedure.java | 3 +-
.../runtime/ConsumerGroupMetaSyncProcedure.java | 3 +-
.../subscription/CreateSubscriptionProcedure.java | 3 +-
.../subscription/DropSubscriptionProcedure.java | 3 +-
.../subscription/topic/AlterTopicProcedure.java | 4 +-
.../subscription/topic/CreateTopicProcedure.java | 21 +-
.../subscription/topic/DropTopicProcedure.java | 3 +-
.../topic/runtime/TopicMetaSyncProcedure.java | 3 +-
.../thrift/ConfigNodeRPCServiceHandler.java | 8 +-
.../thrift/ConfigNodeRPCServiceProcessor.java | 68 +++-
.../iotdb/confignode/persistence/PipeInfoTest.java | 2 +-
.../pipe/plugin/CreatePipePluginProcedureTest.java | 2 +-
.../pipe/plugin/DropPipePluginProcedureTest.java | 2 +-
.../thrift/ConfigNodeRPCServiceProcessorTest.java | 164 ++++++++
.../apache/iotdb/consensus/iot/IoTConsensus.java | 30 +-
.../consensus/iot/IoTConsensusServerImpl.java | 49 ++-
.../consensus/iot/logdispatcher/SyncStatus.java | 60 ++-
.../iot/service/IoTConsensusRPCService.java | 14 +-
.../service/IoTConsensusRPCServiceProcessor.java | 192 ++++------
.../apache/iotdb/consensus/pipe/PipeConsensus.java | 2 +-
.../pipe/service/PipeConsensusRPCService.java | 15 +-
.../service/PipeConsensusRPCServiceProcessor.java | 64 ++--
.../request/PipeTransferTabletRawReq.java | 4 +-
.../common/tablet/PipeRawTabletInsertionEvent.java | 13 +-
.../common/tsfile/PipeTsFileInsertionEvent.java | 14 +-
.../event/realtime/PipeRealtimeEventFactory.java | 2 +-
.../PipeHistoricalDataRegionTsFileExtractor.java | 1 +
.../realtime/assigner/PipeDataRegionAssigner.java | 17 +
.../schemaregion/PipePlanPatternParseVisitor.java | 16 +-
.../visitor/PipeStatementPatternParseVisitor.java | 1 +
.../visitor/PipeStatementToBatchVisitor.java | 1 +
.../iotdb/db/protocol/client/ConfigNodeClient.java | 14 +
.../config/executor/ClusterConfigTaskExecutor.java | 37 +-
.../config/executor/IConfigTaskExecutor.java | 3 +-
.../config/metadata/DropPipePluginTask.java | 6 +-
.../db/queryengine/plan/parser/ASTVisitor.java | 20 +-
.../plan/parser/StatementGenerator.java | 2 +-
.../planner/plan/node/write/InsertTabletNode.java | 1 +
.../metadata/pipe/AlterPipeStatement.java | 9 +
.../metadata/pipe/CreatePipePluginStatement.java | 9 +-
.../metadata/pipe/CreatePipeStatement.java | 9 +
.../metadata/pipe/DropPipePluginStatement.java | 18 +-
.../statement/metadata/pipe/DropPipeStatement.java | 9 +
.../subscription/CreateTopicStatement.java | 10 +-
.../metadata/subscription/DropTopicStatement.java | 9 +
.../db/service/DataNodeServerCommandLine.java | 145 ++++---
.../iotdb/db/service/metrics/WritingMetrics.java | 88 ++++-
.../iotdb/db/storageengine/StorageEngine.java | 4 +
.../storageengine/dataregion/wal/io/LogWriter.java | 7 +
.../dataregion/wal/io/WALByteBufReader.java | 40 +-
.../dataregion/wal/io/WALInputStream.java | 69 +++-
.../dataregion/wal/recover/WALNodeRecoverTask.java | 11 +
.../dataregion/wal/utils/WALInsertNodeCache.java | 5 +-
.../agent/SubscriptionBrokerAgent.java | 17 +-
.../agent/SubscriptionConsumerAgent.java | 4 +-
.../db/subscription/broker/SubscriptionBroker.java | 129 +++++--
.../broker/SubscriptionPrefetchingQueue.java | 73 ++--
.../broker/SubscriptionPrefetchingTabletQueue.java | 9 +-
.../broker/SubscriptionPrefetchingTsFileQueue.java | 27 +-
.../TsFileDeduplicationBlockingPendingQueue.java | 48 ++-
.../db/subscription/event/SubscriptionEvent.java | 6 +-
.../batch/SubscriptionPipeTsFileEventBatch.java | 7 +
.../receiver/SubscriptionReceiverV1.java | 41 +-
.../SubscriptionConnectorSubtaskLifeCycle.java | 2 +-
.../iotdb/db/tools/utils/TsFileSequenceScan.java | 168 +++++++++
.../iotdb/db/tools/utils/TsFileValidationScan.java | 397 +++++++++++++++++++
.../db/tools/validate/TsFileValidationTool.java | 419 ++-------------------
.../apache/iotdb/db/utils/TypeInferenceUtils.java | 2 +-
.../node/write/InsertTabletNodeSerdeTest.java | 35 ++
.../db/service/DataNodeServerCommandLineTest.java | 218 +++++++++++
.../compaction/AbstractCompactionTest.java | 4 +-
.../FastCrossCompactionPerformerTest.java | 4 +-
...eCompactionWithFastPerformerValidationTest.java | 2 +-
...actionWithReadPointPerformerValidationTest.java | 2 +-
.../wal/compression/WALCompressionTest.java | 32 ++
.../iotdb/db/tools/TsFileValidationScanTest.java | 204 ++++++++++
.../iotdb/db/utils/TypeInferenceUtilsTest.java | 6 +-
.../apache/iotdb/commons/conf/CommonConfig.java | 23 ++
.../iotdb/commons/conf/CommonDescriptor.java | 11 +
.../iotdb/commons/pipe/config/PipeConfig.java | 10 +
.../iotdb/commons/pipe/event/EnrichedEvent.java | 19 +-
.../iotdb/commons/service/ThriftService.java | 15 +-
.../subscription/config/SubscriptionConfig.java | 5 +
.../src/main/thrift/confignode.thrift | 21 ++
library-udf/pom.xml | 7 -
.../apache/iotdb/library/dprofile/UDTFSample.java | 24 +-
.../apache/iotdb/library/dprofile/util/Area.java | 62 +++
.../apache/iotdb/library/dprofile/util/Bucket.java | 85 +++++
.../library/dprofile/util/LTThreeBuckets.java | 57 +++
.../library/dprofile/util/OnePassBucketizer.java | 68 ++++
.../library/dprofile/util/SlidingCollector.java | 111 ++++++
.../iotdb/library/dprofile/util/Triangle.java | 59 +++
pom.xml | 63 ++--
133 files changed, 3762 insertions(+), 1121 deletions(-)
create mode 100644
integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeConditionalOperationsIT.java
copy
integration-test/src/test/java/org/apache/iotdb/pipe/it/single/{IoTDBPipeSingleEnvDemoIT.java
=> IoTDBPipeOPCUAIT.java} (74%)
create mode 100644
integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionTopicIT.java
create mode 100644
iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessorTest.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/utils/TsFileSequenceScan.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/utils/TsFileValidationScan.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/service/DataNodeServerCommandLineTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/tools/TsFileValidationScanTest.java
create mode 100644
library-udf/src/main/java/org/apache/iotdb/library/dprofile/util/Area.java
create mode 100644
library-udf/src/main/java/org/apache/iotdb/library/dprofile/util/Bucket.java
create mode 100644
library-udf/src/main/java/org/apache/iotdb/library/dprofile/util/LTThreeBuckets.java
create mode 100644
library-udf/src/main/java/org/apache/iotdb/library/dprofile/util/OnePassBucketizer.java
create mode 100644
library-udf/src/main/java/org/apache/iotdb/library/dprofile/util/SlidingCollector.java
create mode 100644
library-udf/src/main/java/org/apache/iotdb/library/dprofile/util/Triangle.java