This is an automated email from the ASF dual-hosted git repository. caogaofei pushed a commit to branch beyyes/debug-table in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 0557f13942e95b1cf6f4f8a2d80bc8a90da00dc4 Merge: aca76caddf8 47909acb6a3 Author: Beyyes <[email protected]> AuthorDate: Fri Apr 19 12:24:01 2024 +0800 merge .github/workflows/sonar-codecov.yml | 2 +- README.md | 3 + README_ZH.md | 1 + .../pass/IoTDBRegionMigrateClusterCrashIT.java | 3 +- .../pass/IoTDBRegionMigrateConfigNodeCrashIT.java | 2 +- .../pipe/it/autocreate/IoTDBPipeProtocolIT.java | 1 + .../api/customizer/parameter/PipeParameters.java | 77 ++-- .../client/async/AsyncDataNodeClientPool.java | 3 +- .../client/async/handlers/AsyncClientHandler.java | 11 +- .../heartbeat/ConfigNodeHeartbeatHandler.java | 22 +- .../heartbeat/DataNodeHeartbeatHandler.java | 74 ++-- .../handlers/rpc/TransferLeaderRPCHandler.java | 89 +++++ .../client/sync/SyncDataNodeClientPool.java | 9 +- .../iotdb/confignode/manager/ConfigManager.java | 33 +- .../iotdb/confignode/manager/ProcedureManager.java | 10 +- .../iotdb/confignode/manager/load/LoadManager.java | 141 +++---- .../manager/load/balancer/RouteBalancer.java | 412 +++++++++++++-------- .../router/priority/GreedyPriorityBalancer.java | 4 +- .../router/priority/LeaderPriorityBalancer.java | 4 +- .../load/cache/AbstractHeartbeatSample.java} | 33 +- .../manager/load/cache/AbstractLoadCache.java | 86 +++++ .../manager/load/cache/AbstractStatistics.java | 49 +++ .../confignode/manager/load/cache/LoadCache.java | 291 ++++++--------- .../load/cache/consensus/ConsensusGroupCache.java | 61 +++ .../consensus/ConsensusGroupHeartbeatSample.java} | 23 +- .../cache/consensus/ConsensusGroupStatistics.java | 65 ++++ .../manager/load/cache/node/BaseNodeCache.java | 106 +----- .../load/cache/node/ConfigNodeHeartbeatCache.java | 39 +- .../load/cache/node/DataNodeHeartbeatCache.java | 37 +- .../load/cache/node/NodeHeartbeatSample.java | 70 ++-- .../manager/load/cache/node/NodeStatistics.java | 94 +---- .../manager/load/cache/region/RegionCache.java | 69 ++-- .../load/cache/region/RegionGroupCache.java | 104 ++---- .../load/cache/region/RegionGroupStatistics.java | 80 ++-- .../load/cache/region/RegionHeartbeatSample.java | 27 +- .../load/cache/region/RegionStatistics.java | 38 +- .../manager/load/cache/route/RegionRouteCache.java | 147 -------- .../manager/load/service/EventService.java | 272 ++++++++++++++ .../manager/load/service/HeartbeatService.java | 26 +- .../manager/load/service/StatisticsService.java | 237 +----------- ...va => ConsensusGroupStatisticsChangeEvent.java} | 30 +- .../load/subscriber/IClusterStatusSubscriber.java | 8 +- .../subscriber/NodeStatisticsChangeEvent.java} | 32 +- ....java => RegionGroupStatisticsChangeEvent.java} | 24 +- .../iotdb/confignode/manager/node/NodeManager.java | 13 +- .../protocol/IoTDBConfigRegionAirGapConnector.java | 38 +- .../protocol/IoTDBConfigRegionConnector.java | 78 ++-- .../runtime/PipeLeaderChangeHandler.java | 41 +- .../runtime/PipeRuntimeCoordinator.java | 23 +- .../pipe/event/PipeConfigRegionSnapshotEvent.java | 3 +- .../pipe/extractor/IoTDBConfigRegionExtractor.java | 17 +- .../procedure/env/ConfigNodeProcedureEnv.java | 45 +-- .../procedure/env/RegionMaintainHandler.java | 65 ++-- .../impl/node/AddConfigNodeProcedure.java | 10 +- .../impl/region/AddRegionPeerProcedure.java | 53 ++- .../impl/region/CreateRegionGroupsProcedure.java | 78 ++-- .../impl/region/RegionMigrateProcedure.java | 41 +- .../impl/region/RemoveRegionPeerProcedure.java | 42 ++- .../impl/schema/DeleteDatabaseProcedure.java | 5 +- .../procedure/state/RegionTransitionState.java | 1 - .../procedure/state/RemoveRegionPeerState.java | 1 + .../router/priority/GreedyPriorityTest.java | 10 +- .../priority/LeaderPriorityBalancerTest.java | 8 +- ...CacheTest.java => ConsensusGroupCacheTest.java} | 20 +- .../manager/load/cache/NodeCacheTest.java | 37 +- .../manager/load/cache/RegionGroupCacheTest.java | 116 +++--- .../persistence/node/NodeStatisticsTest.java | 66 ---- .../statistics/RegionGroupStatisticsTest.java | 76 ---- .../partition/statistics/RegionStatisticsTest.java | 66 ---- .../org/apache/iotdb/consensus/IConsensus.java | 8 + .../iotdb/consensus/config/IoTConsensusConfig.java | 19 +- .../apache/iotdb/consensus/iot/IoTConsensus.java | 19 +- .../consensus/iot/IoTConsensusServerImpl.java | 55 ++- .../consensus/iot/logdispatcher/LogDispatcher.java | 27 +- .../iot/snapshot/IoTConsensusRateLimiter.java | 59 +++ .../iot/snapshot/SnapshotFragmentReader.java | 4 + .../iotdb/consensus/ratis/RatisConsensus.java | 12 + .../iotdb/consensus/simple/SimpleConsensus.java | 5 + .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 10 + .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 51 ++- .../db/consensus/DataRegionConsensusImpl.java | 2 + ...IoTDBThriftSyncPipeTransferBatchReqBuilder.java | 17 +- .../builder/PipeTransferBatchReqBuilder.java | 32 +- .../airgap/IoTDBDataNodeAirGapConnector.java | 26 +- .../airgap/IoTDBDataRegionAirGapConnector.java | 86 ++++- .../airgap/IoTDBSchemaRegionAirGapConnector.java | 22 +- .../protocol/legacy/IoTDBLegacyPipeConnector.java | 84 ++++- .../connector/protocol/opcua/OpcUaConnector.java | 52 ++- .../async/IoTDBDataRegionAsyncConnector.java | 98 +++-- .../PipeTransferTabletBatchEventHandler.java | 28 +- .../PipeTransferTsFileInsertionEventHandler.java | 26 +- .../thrift/sync/IoTDBDataNodeSyncConnector.java | 39 +- .../thrift/sync/IoTDBDataRegionSyncConnector.java | 90 ++++- .../thrift/sync/IoTDBSchemaRegionConnector.java | 33 +- .../protocol/writeback/WriteBackConnector.java | 51 ++- .../schema/PipeSchemaRegionSnapshotEvent.java | 3 +- .../db/pipe/event/realtime/PipeRealtimeEvent.java | 45 +-- .../schemaregion/IoTDBSchemaRegionExtractor.java | 16 +- .../processor/aggregate/AggregateProcessor.java | 106 ++++-- .../aggregate/TimeSeriesRuntimeState.java | 31 +- .../window/datastructure/TimeSeriesWindow.java | 42 ++- .../window/datastructure/WindowOutput.java | 6 +- .../processor/TumblingWindowingProcessor.java | 12 +- .../impl/DataNodeInternalRPCServiceImpl.java | 52 ++- .../fragment/FragmentInstanceContext.java | 1 + .../source/relational/TableScanOperator.java | 17 +- .../plan/planner/TableOperatorGenerator.java | 93 ++++- .../plan/relational/analyzer/Analysis.java | 29 +- .../relational/metadata/TableMetadataImpl.java | 7 +- .../plan/relational/planner/LogicalPlanner.java | 20 +- .../plan/relational/planner/QueryPlanner.java | 4 +- .../plan/relational/planner/RelationPlanner.java | 17 +- .../relational/planner/RelationalModelPlanner.java | 2 +- .../distribute/FragmentInstanceGenerator.java | 10 +- .../distribute/RelationalDistributionPlanner.java | 40 ++ .../TableModelTypeProviderExtractor.java | 3 + .../plan/relational/planner/node/OutputNode.java | 6 +- .../relational/planner/node/TableScanNode.java | 17 +- .../planner/optimizations/IndexScan.java | 117 +++++- .../relational/planner/optimizations/Util.java | 124 +++++++ .../iotdb/db/service/RegionMigrateService.java | 3 +- .../compaction/schedule/CompactionTaskManager.java | 9 + .../compaction/schedule/CompactionWorker.java | 12 +- .../db/storageengine/rescon/disk/TierManager.java | 18 +- .../storageengine/rescon/quotas/QuotaLimiter.java | 3 + .../broker/SerializedEnrichedEvent.java | 8 +- .../org/apache/iotdb/db/utils/DateTimeUtils.java | 27 -- .../plan/relational/analyzer/AnalyzerTest.java | 12 +- .../plan/relational/analyzer/TestMatadata.java | 29 +- .../metrics/metricsets/system/SystemMetrics.java | 147 ++++++-- .../apache/iotdb/metrics/utils/FileStoreUtils.java | 65 ++++ .../apache/iotdb/metrics/utils/SystemMetric.java | 1 + .../resources/conf/iotdb-common.properties | 7 +- .../apache/iotdb/commons/cluster/RegionStatus.java | 12 +- .../iotdb/commons/concurrent/ThreadName.java | 1 + .../config/constant/PipeExtractorConstant.java | 2 +- .../pipe/connector/PipeReceiverStatusHandler.java | 21 +- .../iotdb/commons/pipe/event/EnrichedEvent.java | 74 ++-- .../extractor/IoTDBNonDataRegionExtractor.java | 4 +- .../commons/pipe/task/subtask/PipeSubtask.java | 14 +- .../quotas/AverageIntervalRateLimiter.java | 2 +- .../commons}/quotas/FixedIntervalRateLimiter.java | 2 +- .../apache/iotdb/commons}/quotas/RateLimiter.java | 6 +- .../iotdb/commons/utils/CommonDateTimeUtils.java | 47 +++ .../commons/utils/CommonDateTimeUtilsTest.java | 41 ++ .../src/main/thrift/datanode.thrift | 8 +- pom.xml | 2 +- 147 files changed, 3794 insertions(+), 2460 deletions(-) diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java index fa7302837a7,52ae6220dba..a3e54709be1 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java @@@ -260,9 -237,10 +260,12 @@@ public class TableMetadataImpl implemen QualifiedObjectName tableName, List<Expression> expressionList, List<String> attributeColumns) { + // fixme, perfect the real metadata impl List<DeviceEntry> result = new ArrayList<>(); - IDeviceID deviceID = new StringArrayDeviceID("beijing", "a_1"); - IDeviceID deviceID = new StringArrayDeviceID("db", "table1", "beijing", "a_1"); -- result.add(new DeviceEntry(deviceID, Arrays.asList("new", "low"))); ++ IDeviceID deviceID1 = new StringArrayDeviceID("db", "table1", "beijing", "a_1"); ++ IDeviceID deviceID2 = new StringArrayDeviceID("db", "table1", "beijing", "b_1"); ++ result.add(new DeviceEntry(deviceID1, Arrays.asList("old", "low"))); ++ result.add(new DeviceEntry(deviceID2, Arrays.asList("new", "high"))); return result; }
