This is an automated email from the ASF dual-hosted git repository.
Caideyipi pushed a change to branch cp-feaeaa6-dev-1.3
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from a750bfd9d8b Merge dev/1.3 into cp-feaeaa6-dev-1.3
add 85f82fd1030 [Pipe] Fix historical TsFile schema recovery race (#18314)
(#18378)
add c77b0ff1586 [Pipe] Batch OPC UA sink writes (#18380)
add cdc95b40391 Pipe: Handle hybrid meta progress indexes (#18331) (#18383)
add 5b0894730cb Pipe: Respect history enable with source time range
(#18362) (#18385)
add 49d6dad7679 [to dev/1.3] Fix IndexOutOfBoundsException caused by
concurrent sort and delete on TVList- #18390
add dcb31e7842b [To dev/1.3] Supply the max_schema/data_region_group_num
param to modify schema when create or alter database (#18391)
add 60b18800b6a [To dev/1.3] clone partial columns of aligned tvlist for
query (#18394)
add 8a84595c271 [To dev/1.3] Optimize aligned memtable flush without
deleted measurements (#18416)
add 34d38ef57d5 Fix pipe tablet memory resize admission limit (#18413)
(#18420)
add fa8efc20e90 Reduce query execution warning log size (#18415)
add e7d97e60dd1 Reduce AirGap receiver log noise (#18424) (#18427)
add 251cf340b71 Optimize aligned row and tablet memory estimation (#18426)
add 8915a2e94c8 [Load] Isolate reused TsFile parser memory from Pipe pool
(#18419) (#18436)
add 28d810c752c fix(jdbc): prevent BigDecimal DoS in
StringUtils.consistentToString (#17508) (#18445)
add 79c94598031 [To dev/1.3] Pipe: Handle transient resource failures
locally (#18381) (#18430)
add 505b18a8af3 fix(pipe): reuse safe local progress on alter (#18423)
(#18444)
add 2162b8d7bec [To dev/1.3] Update thrift to 0.23.0 (#18446)
add fb0b2d9301f [Pipe] Support configurable TsFile parser (#18440) (#18449)
add 5a58b8ba427 [to dev/1.3] Optimize load logic (#18153) (#18459)
add 9e2074d26bd [to dev/1.3] Optimize aligned TVList partial-column clone
memory usage (#18454)
add 534f1e2d83d [to dev/1.3] Remove incorrect timeout reassignment after
query planning (#18196)
add 6e8a31ad2d6 [to dev/1.3] Improve query modification loading memory
control (#17788) (#18466)
add a177a93017c [To dev/1.3] Handle invalid DataNode id in region
operations (#18478)
add 8ccd2347eb5 Merge dev/1.3 into cp-feaeaa6-dev-1.3
No new revisions were added by this update.
Summary of changes:
LICENSE-binary | 2 +-
integration-test/pom.xml | 10 +
.../IoTDBDatabaseAutoDataRegionGroupQuotaIT.java | 122 +++
.../IoTDBDatabaseMixedRegionGroupPolicyIT.java} | 44 +-
.../it/database/IoTDBDatabaseRegionControlIT.java | 135 ++-
.../database/IoTDBSystemDatabaseRegionGroupIT.java | 90 ++
.../IoTDBRegionGroupExpandAndShrinkForIoTV1IT.java | 48 ++
.../commit/IoTDBRegionReconstructForIoTV1IT.java | 50 ++
.../apache/iotdb/db/it/IoTDBLoadTsFileAuthIT.java | 283 +++++++
.../iotdb/pipe/it/autocreate/IoTDBPipeAlterIT.java | 48 ++
.../it/autocreate/IoTDBPipeAutoConflictIT.java | 5 +
.../pipe/it/autocreate/IoTDBPipeAutoDropIT.java | 2 +-
.../pipe/it/autocreate/IoTDBPipeIdempotentIT.java | 6 +-
.../pipe/it/autocreate/IoTDBPipeSourceIT.java | 60 ++
iotdb-client/jdbc/src/main/feature/feature.xml | 2 +-
.../java/org/apache/iotdb/jdbc/StringUtils.java | 21 +-
.../org/apache/iotdb/jdbc/StringUtilsTest.java | 74 ++
.../iotdb/rpc/TNonblockingSocketWrapper.java | 4 +-
.../java/org/apache/iotdb/rpc/TSocketWrapper.java | 4 +-
.../org/apache/iotdb/db/qp/sql/IdentifierParser.g4 | 6 +-
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 4 +-
.../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 8 +-
.../heartbeat/DataNodeHeartbeatHandler.java | 3 +-
.../response/pipe/task/PipeTableResp.java | 1 +
.../iotdb/confignode/manager/ConfigManager.java | 3 +-
.../iotdb/confignode/manager/ProcedureManager.java | 35 +-
.../manager/partition/PartitionManager.java | 10 +-
.../pipe/agent/task/PipeConfigNodeSubtask.java | 8 +
.../pipe/agent/task/PipeConfigNodeTaskAgent.java | 17 +-
.../runtime/PipeRuntimeCoordinator.java | 7 +-
.../runtime/heartbeat/PipeHeartbeat.java | 27 +
.../runtime/heartbeat/PipeHeartbeatParser.java | 1 +
.../runtime/heartbeat/PipeHeartbeatScheduler.java | 6 +-
.../manager/schema/ClusterSchemaManager.java | 252 ++++--
.../persistence/schema/ClusterSchemaInfo.java | 24 +-
.../procedure/env/ConfigNodeProcedureEnv.java | 33 +
.../impl/pipe/task/AlterPipeProcedureV2.java | 83 +-
.../thrift/ConfigNodeRPCServiceHandler.java | 3 +-
.../thrift/ConfigNodeRPCServiceProcessor.java | 32 +-
.../consensus/response/pipe/PipeTableRespTest.java | 21 +
.../ProcedureManagerRegionOperationTest.java | 216 +++++
.../runtime/heartbeat/PipeHeartbeatParserTest.java | 55 ++
.../thrift/ConfigNodeRPCServiceProcessorTest.java | 34 +
.../iot/service/IoTConsensusRPCServiceHandler.java | 4 +-
.../service/PipeConsensusRPCServiceHandler.java | 4 +-
.../db/pipe/agent/task/PipeDataNodeTaskAgent.java | 102 ++-
.../task/builder/PipeDataNodeTaskBuilder.java | 56 +-
.../subtask/processor/PipeProcessorSubtask.java | 12 +
.../agent/task/subtask/sink/PipeSinkSubtask.java | 8 +
.../tsfile/PipeCompactedTsFileInsertionEvent.java | 1 +
.../common/tsfile/PipeTsFileInsertionEvent.java | 41 +-
.../container/TsFileInsertionDataContainer.java | 40 +-
.../TsFileInsertionDataContainerProvider.java | 53 ++
.../query/TsFileInsertionQueryDataContainer.java | 65 +-
.../TsFileInsertionQueryDataTabletIterator.java | 10 +-
.../scan/TsFileInsertionScanDataContainer.java | 94 ++-
.../TsFileInsertionEventParserMemoryBlock.java | 20 +-
.../TsFileInsertionEventParserMemoryManager.java | 75 ++
.../processor/aggregate/AggregateProcessor.java | 14 +-
.../twostage/plugin/TwoStageCountProcessor.java | 28 +-
.../protocol/airgap/IoTDBAirGapReceiver.java | 6 -
.../protocol/thrift/IoTDBDataNodeReceiver.java | 75 +-
.../db/pipe/resource/memory/PipeMemoryManager.java | 19 +-
.../request/PipeTransferTsFileSealWithModReq.java | 77 +-
.../protocol/airgap/IoTDBDataRegionAirGapSink.java | 11 +-
.../protocol/opcua/client/IoTDBOpcUaClient.java | 174 +++-
.../thrift/async/IoTDBDataRegionAsyncSink.java | 57 +-
.../async/handler/PipeTransferTsFileHandler.java | 10 +-
.../thrift/sync/IoTDBDataRegionSyncSink.java | 10 +-
.../source/dataregion/IoTDBDataRegionSource.java | 25 +-
.../PipeHistoricalDataRegionTsFileSource.java | 48 +-
.../realtime/PipeRealtimeDataRegionSource.java | 9 +
.../realtime/assigner/PipeDataRegionAssigner.java | 1 +
.../thrift/handler/BaseServerContextHandler.java | 8 +-
.../handler/InternalServiceThriftHandler.java | 3 +-
.../impl/DataNodeInternalRPCServiceImpl.java | 18 +
.../thrift/impl/PushMultiPipeMetaHelper.java | 36 +-
.../db/queryengine/common/MPPQueryContext.java | 5 +-
.../common/header/ColumnHeaderConstant.java | 4 +-
.../execution/MemoryEstimationHelper.java | 2 +-
.../MPPDataExchangeServiceThriftHandler.java | 3 +-
.../fragment/FragmentInstanceContext.java | 107 ++-
.../fragment/QueryModificationLoader.java | 306 +++++++
.../execution/schedule/AbstractDriverThread.java | 26 +-
.../plan/analyze/load/LoadTsFileAnalyzer.java | 126 ++-
.../plan/analyze/schema/ISchemaFetcher.java | 19 +
.../plan/analyze/schema/SchemaValidator.java | 34 +-
.../queryengine/plan/execution/QueryExecution.java | 4 -
.../config/metadata/DatabaseSchemaTask.java | 9 +-
.../execution/config/sys/pipe/ShowPipeTask.java | 9 +
.../db/queryengine/plan/parser/ASTVisitor.java | 12 +-
.../memory/FakedMemoryReservationManager.java | 6 +
.../planner/memory/MemoryReservationManager.java | 14 +
.../NotThreadSafeMemoryReservationManager.java | 16 +-
.../memory/ThreadSafeMemoryReservationManager.java | 5 +
.../plan/statement/crud/LoadTsFileStatement.java | 16 +
.../metadata/DatabaseSchemaStatement.java | 28 +-
.../schemaregion/utils/ResourceByPathUtils.java | 343 +++++---
.../memtable/AbstractWritableMemChunk.java | 24 +-
.../memtable/AlignedReadOnlyMemChunk.java | 4 +-
.../memtable/AlignedWritableMemChunk.java | 317 +++++++
.../dataregion/memtable/ReadOnlyMemChunk.java | 4 +-
.../dataregion/memtable/TsFileProcessor.java | 351 ++++----
.../dataregion/modification/ModificationFile.java | 5 +-
.../io/LocalTextModificationAccessor.java | 124 ++-
.../ModificationIterator.java} | 20 +-
.../modification/io/ModificationReader.java | 3 +-
.../load/active/ActiveLoadPathHelper.java | 70 +-
.../load/active/ActiveLoadTsFileLoader.java | 6 +-
.../storageengine/load/active/ActiveLoadUtil.java | 28 +-
.../load/config/LoadTsFileConfigurator.java | 22 +
.../converter/LoadTreeTsFileTabletIterator.java | 23 +-
.../load/memory/LoadTsFileMemoryBlock.java | 22 +-
.../load/memory/LoadTsFileMemoryManager.java | 44 +
.../load/memory/LoadTsFileParserMemoryManager.java | 116 +++
.../db/utils/datastructure/AlignedTVList.java | 917 +++++++++++++++++----
.../db/utils/datastructure/BackAlignedTVList.java | 6 +-
.../db/utils/datastructure/QuickAlignedTVList.java | 6 +-
.../iotdb/db/utils/datastructure/TVList.java | 23 +-
.../db/utils/datastructure/TimAlignedTVList.java | 6 +-
.../pipe/agent/task/PipeDataNodeTaskAgentTest.java | 159 ++++
.../task/builder/PipeDataNodeTaskBuilderTest.java | 79 ++
.../task/subtask/sink/PipeSinkSubtaskTest.java | 49 ++
.../event/TsFileInsertionDataContainerTest.java | 92 ++-
.../plugin/TwoStageCountProcessorTest.java | 55 ++
.../protocol/thrift/IoTDBDataNodeReceiverTest.java | 30 +
.../memory/PipeMemoryManagerResizeTest.java | 24 +
.../pipe/sink/PipeDataNodeThriftRequestTest.java | 30 +
.../opcua/client/IoTDBOpcUaClientTest.java | 176 ++++
.../db/pipe/source/IoTDBDataRegionSourceTest.java | 31 +
.../PipeHistoricalDataRegionTsFileSourceTest.java | 49 ++
...nternalRPCServiceImplPushMultiPipeMetaTest.java | 72 ++
.../fragment/FragmentInstanceExecutionTest.java | 166 ++++
.../fragment/QueryModificationLoaderTest.java | 372 +++++++++
.../schedule/AbstractDriverThreadTest.java | 60 ++
.../plan/analyze/load/LoadTsFileAnalyzerTest.java | 72 +-
.../config/sys/pipe/ShowPipeTaskTest.java | 73 ++
.../LocalExecutionPlannerOperatorsMemoryTest.java | 46 ++
.../utils/ResourceByPathUtilsTest.java | 128 +++
.../TsFileResourceProgressIndexTest.java | 9 +
.../AlignedRowsBranchComparisonBenchmarkTest.java | 286 +++++++
.../dataregion/memtable/MemTableFlushTaskTest.java | 64 ++
.../dataregion/memtable/TsFileProcessorTest.java | 144 +++-
.../load/active/ActiveLoadPathHelperTest.java | 174 ++++
.../LoadTsFileParserPipeMemoryIsolationTest.java | 66 ++
.../load/memory/LoadTsFileMemoryManagerTest.java | 20 +
.../db/utils/datastructure/AlignedTVListTest.java | 285 ++++++-
.../reporter/iotdb/IoTDBSessionReporter.java | 4 +-
.../commons/consensus/index/ProgressIndex.java | 10 +
.../consensus/index/impl/HybridProgressIndex.java | 25 +
.../consensus/index/impl/IoTProgressIndex.java | 9 +
.../consensus/index/impl/MetaProgressIndex.java | 9 +
.../consensus/index/impl/MinimumProgressIndex.java | 9 +
.../consensus/index/impl/RecoverProgressIndex.java | 9 +
.../consensus/index/impl/SimpleProgressIndex.java | 9 +
.../consensus/index/impl/StateProgressIndex.java | 9 +
.../index/impl/TimeWindowStateProgressIndex.java | 11 +-
.../pipe/PipeRuntimeSinkResourceException.java | 51 ++
.../commons/pipe/agent/task/PipeTaskAgent.java | 10 +
.../agent/task/meta/PipeTemporaryMetaInAgent.java | 16 +-
.../task/meta/PipeTemporaryMetaInCoordinator.java | 84 +-
.../task/subtask/PipeAbstractSinkSubtask.java | 28 +-
.../pipe/config/constant/PipeSourceConstant.java | 5 +
.../pipe/config/constant/SystemConstant.java | 2 +
.../options/PipeInclusionOptions.java | 31 +
.../pipe/receiver/PipeReceiverStatusHandler.java | 21 +-
.../pipe/resource/PipeRecentFailureCounter.java | 94 +++
.../pipe/resource/PipeResourceFailureType.java | 18 +-
.../commons/pipe/resource/PipeStopStrategy.java | 76 ++
.../commons/pipe/sink/protocol/IoTDBSink.java | 8 +
.../pipe/source/IoTDBNonDataRegionSource.java | 47 +-
.../iotdb/commons/service/NoopServerContext.java | 48 ++
.../client/mock/MockInternalRPCService.java | 7 +-
.../commons/consensus/index/ProgressIndexTest.java | 82 ++
.../agent/task/meta/PipeTemporaryMetaTest.java | 58 ++
.../resource/PipeRecentFailureCounterTest.java | 48 ++
.../pipe/resource/PipeStopStrategyTest.java | 83 ++
.../pipe/source/IoTDBNonDataRegionSourceTest.java | 135 +++
.../NoopServerContextTest.java} | 34 +-
.../thrift-commons/src/main/thrift/common.thrift | 1 +
.../src/main/thrift/confignode.thrift | 2 +-
.../src/main/thrift/datanode.thrift | 3 +-
pom.xml | 45 +-
183 files changed, 9229 insertions(+), 1071 deletions(-)
create mode 100644
integration-test/src/test/java/org/apache/iotdb/confignode/it/database/IoTDBDatabaseAutoDataRegionGroupQuotaIT.java
copy
integration-test/src/test/java/org/apache/iotdb/{db/it/aligned/IoTDBInsertAlignedValues4IT.java
=> confignode/it/database/IoTDBDatabaseMixedRegionGroupPolicyIT.java} (59%)
create mode 100644
integration-test/src/test/java/org/apache/iotdb/confignode/it/database/IoTDBSystemDatabaseRegionGroupIT.java
create mode 100644
integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileAuthIT.java
create mode 100644
iotdb-client/jdbc/src/test/java/org/apache/iotdb/jdbc/StringUtilsTest.java
create mode 100644
iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ProcedureManagerRegionOperationTest.java
copy
iotdb-client/isession/src/main/java/org/apache/iotdb/isession/INodeSupplier.java
=>
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParserMemoryBlock.java
(63%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParserMemoryManager.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryModificationLoader.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/{package-info.java
=> io/ModificationIterator.java} (67%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/memory/LoadTsFileParserMemoryManager.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/opcua/client/IoTDBOpcUaClientTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryModificationLoaderTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/schedule/AbstractDriverThreadTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/ShowPipeTaskTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtilsTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedRowsBranchComparisonBenchmarkTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadPathHelperTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/converter/LoadTsFileParserPipeMemoryIsolationTest.java
create mode 100644
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/exception/pipe/PipeRuntimeSinkResourceException.java
create mode 100644
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/resource/PipeRecentFailureCounter.java
copy
integration-test/src/test/java/org/apache/iotdb/subscription/it/DisplayName.java
=>
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/resource/PipeResourceFailureType.java
(72%)
create mode 100644
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/resource/PipeStopStrategy.java
create mode 100644
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/NoopServerContext.java
create mode 100644
iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/consensus/index/ProgressIndexTest.java
create mode 100644
iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeTemporaryMetaTest.java
create mode 100644
iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/resource/PipeRecentFailureCounterTest.java
create mode 100644
iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/resource/PipeStopStrategyTest.java
create mode 100644
iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSourceTest.java
copy
iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/{udf/utils/KDTreeUtilTest.java
=> service/NoopServerContextTest.java} (53%)
