This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch MemoryControl in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit ec62d34390549d74e700d2698e373a2d050b7e0b Merge: ba55d2e35e 5faa9da049 Author: JackieTien97 <[email protected]> AuthorDate: Mon Aug 15 09:51:16 2022 +0800 Merge remote-tracking branch 'origin/master' into MemoryControl .../org/apache/iotdb/db/qp/sql/IdentifierParser.g4 | 1 + .../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 10 +- .../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 4 + client-py/README.md | 1 + .../resources/conf/iotdb-confignode.properties | 29 +- .../confignode/client/DataNodeRequestType.java | 5 +- .../sync/datanode/SyncDataNodeClientPool.java | 38 +- .../statemachine/PartitionRegionStateMachine.java | 8 +- .../iotdb/confignode/manager/ConfigManager.java | 17 +- .../apache/iotdb/confignode/manager/IManager.java | 9 +- .../iotdb/confignode/manager/NodeManager.java | 281 +++++++++--- .../iotdb/confignode/manager/PartitionManager.java | 52 ++- .../iotdb/confignode/manager/load/LoadManager.java | 460 +------------------ .../manager/load/LoadManagerMetrics.java | 266 +++++++++++ .../manager/load/balancer/RegionBalancer.java | 10 +- .../manager/load/balancer/RouteBalancer.java | 23 +- .../load/heartbeat/ConfigNodeHeartbeatCache.java | 8 +- .../load/heartbeat/DataNodeHeartbeatCache.java | 2 +- .../iotdb/confignode/persistence/NodeInfo.java | 47 +- .../persistence/partition/PartitionInfo.java | 67 ++- .../partition/StorageGroupPartitionTable.java | 91 ++-- .../procedure/env/ConfigNodeProcedureEnv.java | 30 +- .../procedure/env/DataNodeRemoveHandler.java | 139 +++++- .../procedure/impl/RegionMigrateProcedure.java | 48 +- .../procedure/impl/RemoveDataNodeProcedure.java | 26 +- .../procedure/state/RegionTransitionState.java | 5 +- .../iotdb/confignode/service/ConfigNode.java | 8 +- .../confignode/service/ConfigNodeCommandLine.java | 2 +- .../service/thrift/ConfigNodeRPCService.java | 5 +- .../thrift/ConfigNodeRPCServiceHandler.java | 8 +- .../thrift/ConfigNodeRPCServiceProcessor.java | 10 +- .../resources/confignode1conf/iotdb-metric.yml | 2 +- .../resources/confignode2conf/iotdb-metric.yml | 2 +- .../confignode3conf/iotdb-confignode.properties | 2 +- .../resources/confignode3conf/iotdb-metric.yml | 2 +- ...uest.java => BatchIndexedConsensusRequest.java} | 53 +-- .../common/request/IndexedConsensusRequest.java | 33 +- .../common/response/ConsensusWriteResponse.java | 17 + .../iotdb/consensus/config/MultiLeaderConfig.java | 54 ++- .../multileader/MultiLeaderServerImpl.java | 51 ++- .../multileader/logdispatcher/LogDispatcher.java | 45 +- .../service/MultiLeaderRPCServiceProcessor.java | 52 +-- .../multileader/wal/ConsensusReqReader.java | 3 + .../multileader/logdispatcher/SyncStatusTest.java | 4 +- .../multileader/util/FakeConsensusReqReader.java | 5 + .../multileader/util/TestStateMachine.java | 40 +- .../UserGuide/API/Programming-Python-Native-API.md | 3 +- docs/UserGuide/Cluster/Cluster-Setup.md | 279 +++++++++--- .../Maintenance-Tools/Maintenance-Command.md | 248 ++++++----- docs/UserGuide/Maintenance-Tools/Metric-Tool.md | 4 +- docs/UserGuide/Query-Data/Aggregate-Query.md | 94 ++++ docs/UserGuide/UDF-Library/Data-Repairing.md | 61 +++ docs/UserGuide/UDF-Library/Series-Processing.md | 40 +- .../UserGuide/API/Programming-Python-Native-API.md | 5 +- docs/zh/UserGuide/Cluster/Cluster-Setup.md | 270 ++++++++++-- .../Maintenance-Tools/Maintenance-Command.md | 242 +++++----- docs/zh/UserGuide/Maintenance-Tools/Metric-Tool.md | 4 +- docs/zh/UserGuide/Query-Data/Aggregate-Query.md | 93 ++++ docs/zh/UserGuide/UDF-Library/Data-Repairing.md | 62 +++ docs/zh/UserGuide/UDF-Library/Series-Processing.md | 40 +- .../Apache IoTDB ConfigNode Dashboard v0.14.0.json | 375 +++++++++++++++- .../java/org/apache/iotdb/it/env/AbstractEnv.java | 33 ++ .../BuiltinTimeSeriesGeneratingFunctionEnum.java | 5 +- .../apache/iotdb/db/it/groupby/IoTDBHavingIT.java | 247 +++++++++++ .../db/it/udf/IoTDBUDTFBuiltinFunctionIT.java | 210 ++++++++- .../src/test/resources/logback-test.xml | 2 +- .../iotdb/db/integration/IoTDBCheckConfigIT.java | 5 +- .../db/integration/sync/IoTDBSyncReceiverIT.java | 6 +- integration/src/test/resources/logback-test.xml | 2 +- .../apache/iotdb/jdbc/IoTDBDatabaseMetadata.java | 36 ++ metrics/ReadMe.md | 103 ++--- metrics/dropwizard-metrics/pom.xml | 4 +- .../dropwizard/DropwizardMetricManager.java | 375 ++-------------- .../dropwizard/DropwizardMetricNameTool.java | 66 +++ .../iotdb/metrics/dropwizard/MetricName.java | 174 -------- .../reporter/DropwizardIoTDBReporter.java | 34 +- .../dropwizard/reporter/DropwizardJmxReporter.java | 14 +- .../{MetricType.java => DropwizardMetricType.java} | 4 +- .../reporter/DropwizardMetricsExporter.java | 71 ++- .../reporter/DropwizardPrometheusReporter.java | 15 +- .../metrics/dropwizard/reporter/IoTDBReporter.java | 31 +- .../dropwizard/reporter/PrometheusTextWriter.java | 2 +- ...org.apache.iotdb.metrics.AbstractMetricManager} | 0 .../dropwizard/DropwizardMetricManagerTest.java | 339 -------------- .../main/assembly/resources/conf/iotdb-metric.yml | 3 + .../iotdb/metrics/AbstractMetricManager.java | 490 +++++++++++++++++++++ .../iotdb/metrics/AbstractMetricService.java | 262 +++++++++++ .../iotdb/metrics/DoNothingMetricService.java | 8 +- .../org/apache/iotdb/metrics/MetricManager.java | 170 ------- .../org/apache/iotdb/metrics/MetricService.java | 167 ------- .../apache/iotdb/metrics/config/MetricConfig.java | 35 +- .../metrics/config/MetricConfigDescriptor.java | 109 +++-- .../apache/iotdb/metrics/config/ReloadLevel.java | 5 + .../apache/iotdb/metrics/impl/DoNothingGauge.java | 8 +- .../iotdb/metrics/impl/DoNothingMetricManager.java | 111 +---- .../iotdb/metrics/predefined/IMetricSet.java | 11 +- .../{utils => predefined}/PredefinedMetric.java | 2 +- .../predefined/jvm/JvmClassLoaderMetrics.java | 6 +- .../metrics/predefined/jvm/JvmCompileMetrics.java | 6 +- .../iotdb/metrics/predefined/jvm/JvmGcMetrics.java | 7 +- .../metrics/predefined/jvm/JvmMemoryMetrics.java | 7 +- .../iotdb/metrics/predefined/jvm/JvmMetrics.java | 6 +- .../metrics/predefined/jvm/JvmThreadMetrics.java | 6 +- .../{utils => predefined/jvm}/JvmUtils.java | 4 +- .../metrics/predefined/logback/LogbackMetrics.java | 11 +- .../iotdb/metrics/reporter/CompositeReporter.java | 31 +- .../apache/iotdb/metrics/reporter/Reporter.java | 12 +- .../org/apache/iotdb/metrics/type/Counter.java | 7 +- .../java/org/apache/iotdb/metrics/type/Gauge.java | 9 +- .../org/apache/iotdb/metrics/type/Histogram.java | 3 +- .../iotdb/metrics/type/HistogramSnapshot.java | 26 +- .../org/apache/iotdb/metrics/type/IMetric.java | 1 + .../java/org/apache/iotdb/metrics/type/Rate.java | 11 +- .../java/org/apache/iotdb/metrics/type/Timer.java | 6 +- .../org/apache/iotdb/metrics/utils/MetricInfo.java | 191 ++++++++ .../apache/iotdb/metrics/utils/MetricLevel.java | 2 +- .../{PredefinedMetric.java => MetricType.java} | 12 +- .../iotdb/metrics/config/MetricConfigTest.java | 33 +- .../interface/src/test/resources/iotdb-metric.yml | 35 +- metrics/micrometer-metrics/pom.xml | 6 +- .../iotdb/metrics/micrometer/MetricName.java | 66 --- .../micrometer/MicrometerMetricManager.java | 422 +++--------------- .../reporter/MicrometerIoTDBReporter.java | 6 +- .../micrometer/reporter/MicrometerJmxReporter.java | 6 +- .../reporter/MicrometerPrometheusReporter.java | 8 +- .../metrics/micrometer/type/MicrometerRate.java | 2 +- .../metrics/micrometer/type/MicrometerTimer.java | 2 +- ...org.apache.iotdb.metrics.AbstractMetricManager} | 0 .../micrometer/MicrometerMetricManagerTest.java | 106 ----- .../org/apache/iotdb/commons}/StepTracker.java | 29 +- .../iotdb/commons/client/ClientPoolFactory.java | 13 +- .../iotdb/commons/concurrent/ThreadName.java | 9 +- .../apache/iotdb/commons/service/ServiceType.java | 7 +- .../BuiltinTimeSeriesGeneratingFunction.java | 5 +- ...DTFDeDuplication.java => UDTFChangePoints.java} | 2 +- .../commons/udf/builtin/UDTFMasterRepair.java | 97 ++++ .../apache/iotdb/commons/udf/utils/KDTreeUtil.java | 314 +++++++++++++ .../iotdb/commons/udf/utils/MasterRepairUtil.java | 300 +++++++++++++ pom.xml | 3 +- .../schemaregion/rocksdb/RSchemaRegion.java | 5 +- .../resources/conf/iotdb-datanode.properties | 35 +- .../iotdb/db/auth/ClusterAuthorityFetcher.java | 1 + .../apache/iotdb/db/client/ConfigNodeClient.java | 10 +- .../iotdb/db/client/DataNodeClientPoolFactory.java | 9 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 13 +- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 16 +- .../org/apache/iotdb/db/conf/IoTDBStartCheck.java | 36 ++ .../db/consensus/DataRegionConsensusImpl.java | 4 + .../statemachine/DataRegionStateMachine.java | 159 ++++++- .../org/apache/iotdb/db/engine/StorageEngine.java | 3 + .../apache/iotdb/db/engine/StorageEngineV2.java | 6 + .../apache/iotdb/db/engine/cache/ChunkCache.java | 22 +- .../db/engine/cache/TimeSeriesMetadataCache.java | 54 +-- .../compaction/cross/CrossSpaceCompactionTask.java | 3 +- .../utils/ReadPointCrossCompactionEstimator.java | 44 +- .../utils/SingleSeriesCompactionExecutor.java | 3 - .../impl/ReadPointCompactionPerformer.java | 2 +- .../writer/AbstractCompactionWriter.java | 3 - .../apache/iotdb/db/engine/flush/FlushManager.java | 48 +- .../iotdb/db/engine/flush/MemTableFlushTask.java | 22 +- .../iotdb/db/engine/memtable/AbstractMemTable.java | 139 +++--- .../iotdb/db/engine/storagegroup/DataRegion.java | 26 +- .../engine/storagegroup/TsFileProcessorInfo.java | 42 +- .../db/engine/storagegroup/TsFileResourceList.java | 20 +- .../db/metadata/cache/DataNodeSchemaCache.java | 38 +- .../db/metadata/rescon/TimeseriesStatistics.java | 22 +- .../db/metadata/schemaregion/ISchemaRegion.java | 6 +- .../schemaregion/SchemaRegionMemoryImpl.java | 11 +- .../schemaregion/SchemaRegionSchemaFileImpl.java | 6 +- .../apache/iotdb/db/mpp/execution/QueryState.java | 2 +- .../iotdb/db/mpp/execution/QueryStateMachine.java | 8 +- .../mpp/execution/exchange/LocalSourceHandle.java | 4 +- .../execution/exchange/MPPDataExchangeManager.java | 4 + .../execution/exchange/MPPDataExchangeService.java | 5 +- .../MPPDataExchangeServiceThriftHandler.java | 8 +- .../db/mpp/execution/exchange/SinkHandle.java | 4 +- .../db/mpp/execution/exchange/SourceHandle.java | 2 +- .../apache/iotdb/db/mpp/plan/analyze/Analysis.java | 21 + .../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 183 +++++++- .../db/mpp/plan/analyze/ClusterSchemaFetcher.java | 134 +++--- .../db/mpp/plan/analyze/ConcatPathRewriter.java | 7 + .../db/mpp/plan/analyze/ExpressionAnalyzer.java | 164 +++++-- .../iotdb/db/mpp/plan/analyze/ExpressionUtils.java | 107 ++++- .../db/mpp/plan/analyze/FakeSchemaFetcherImpl.java | 6 +- .../iotdb/db/mpp/plan/analyze/ISchemaFetcher.java | 6 +- .../iotdb/db/mpp/plan/analyze/SchemaValidator.java | 2 +- .../mpp/plan/analyze/StandaloneSchemaFetcher.java | 18 +- .../db/mpp/plan/execution/QueryExecution.java | 23 +- .../config/executor/ClusterConfigTaskExecutor.java | 89 ++-- .../config/executor/IConfigTaskExecutor.java | 8 +- .../executor/StandaloneConfigTaskExecutor.java | 17 +- .../plan/execution/config/sys/ClearCacheTask.java | 13 +- .../mpp/plan/execution/config/sys/FlushTask.java | 10 +- .../mpp/plan/execution/config/sys/MergeTask.java | 12 +- .../iotdb/db/mpp/plan/expression/Expression.java | 28 +- .../plan/expression/binary/AdditionExpression.java | 10 - .../plan/expression/binary/BinaryExpression.java | 110 ----- .../plan/expression/binary/DivisionExpression.java | 10 - .../plan/expression/binary/EqualToExpression.java | 9 - .../expression/binary/GreaterEqualExpression.java | 10 - .../expression/binary/GreaterThanExpression.java | 10 - .../expression/binary/LessEqualExpression.java | 9 - .../plan/expression/binary/LessThanExpression.java | 9 - .../plan/expression/binary/LogicAndExpression.java | 9 - .../plan/expression/binary/LogicOrExpression.java | 9 - .../plan/expression/binary/ModuloExpression.java | 9 - .../binary/MultiplicationExpression.java | 10 - .../plan/expression/binary/NonEqualExpression.java | 9 - .../expression/binary/SubtractionExpression.java | 10 - .../mpp/plan/expression/leaf/ConstantOperand.java | 42 -- .../plan/expression/leaf/TimeSeriesOperand.java | 62 --- .../mpp/plan/expression/leaf/TimestampOperand.java | 60 --- .../plan/expression/multi/FunctionExpression.java | 236 +--------- .../plan/expression/ternary/BetweenExpression.java | 15 - .../plan/expression/ternary/TernaryExpression.java | 130 ------ .../db/mpp/plan/expression/unary/InExpression.java | 8 - .../plan/expression/unary/IsNullExpression.java | 8 - .../mpp/plan/expression/unary/LikeExpression.java | 8 - .../plan/expression/unary/LogicNotExpression.java | 8 - .../plan/expression/unary/NegationExpression.java | 8 - .../plan/expression/unary/RegularExpression.java | 8 - .../mpp/plan/expression/unary/UnaryExpression.java | 86 ---- .../visitor/IntermediateLayerVisitor.java | 454 +++++++++++++++++++ .../visitor/OldIntermediateLayerVisitor.java | 461 +++++++++++++++++++ .../iotdb/db/mpp/plan/parser/ASTVisitor.java | 23 + .../db/mpp/plan/planner/LogicalPlanVisitor.java | 97 +++- .../plan/planner/plan/node/write/InsertNode.java | 4 + .../planner/plan/node/write/InsertRowNode.java | 9 + .../db/mpp/plan/scheduler/ClusterScheduler.java | 2 +- .../scheduler/FragmentInstanceDispatcherImpl.java | 11 +- .../plan/statement/component/HavingCondition.java | 29 +- .../db/mpp/plan/statement/crud/QueryStatement.java | 47 +- .../mpp/transformation/dag/builder/DAGBuilder.java | 24 +- .../dag/builder/EvaluationDAGBuilder.java | 21 +- .../dag/memory/LayerMemoryAssigner.java | 10 +- .../apache/iotdb/db/qp/executor/PlanExecutor.java | 27 +- .../db/query/pool/RawQueryReadTaskPoolManager.java | 47 +- .../java/org/apache/iotdb/db/service/DataNode.java | 31 +- .../db/service/DataNodeInternalRPCService.java | 5 +- .../java/org/apache/iotdb/db/service/IoTDB.java | 12 +- .../apache/iotdb/db/service/IoTDBShutdownHook.java | 2 - .../java/org/apache/iotdb/db/service/NewIoTDB.java | 12 +- .../org/apache/iotdb/db/service/RPCService.java | 5 +- .../iotdb/db/service/RegionMigrateService.java | 387 +++++++++------- .../{MetricsService.java => MetricService.java} | 20 +- ...csServiceMBean.java => MetricServiceMBean.java} | 2 +- .../db/service/metrics/predefined/FileMetrics.java | 217 +++++---- .../service/metrics/predefined/ProcessMetrics.java | 14 +- .../service/metrics/predefined/SystemMetrics.java | 69 ++- .../metrics/recorder/CacheMetricsRecorder.java | 36 +- .../recorder/CompactionMetricsRecorder.java | 42 +- .../db/service/thrift/ProcessorWithMetrics.java | 5 +- .../handler/InternalServiceThriftHandler.java | 8 +- .../thrift/handler/RPCServiceThriftHandler.java | 8 +- .../service/thrift/impl/ClientRPCServiceImpl.java | 8 +- .../impl/DataNodeInternalRPCServiceImpl.java | 83 ++-- .../db/service/thrift/impl/TSServiceImpl.java | 8 +- .../SenderService.java => SyncService.java} | 213 ++++++--- .../iotdb/db/sync/common/ISyncInfoFetcher.java | 4 +- .../iotdb/db/sync/common/LocalSyncInfoFetcher.java | 7 +- .../org/apache/iotdb/db/sync/common/SyncInfo.java | 5 + .../db/sync/common/persistence/SyncLogReader.java | 2 +- .../iotdb/db/sync/receiver/ReceiverService.java | 151 ------- .../org/apache/iotdb/db/sync/sender/pipe/Pipe.java | 4 +- .../transport/client/IoTDBSInkTransportClient.java | 6 +- .../transport/server/TransportServerManager.java | 7 +- .../server/TransportServerThriftHandler.java | 8 +- .../java/org/apache/iotdb/db/wal/io/WALReader.java | 11 +- .../org/apache/iotdb/db/wal/node/WALFakeNode.java | 5 + .../java/org/apache/iotdb/db/wal/node/WALNode.java | 15 +- .../iotdb/db/wal/recover/WALNodeRecoverTask.java | 8 +- .../engine/compaction/AbstractCompactionTest.java | 6 + .../cross/CrossSpaceCompactionValidationTest.java | 1 + .../apache/iotdb/db/metric/MetricServiceTest.java | 276 ++++++++++++ .../datanode1conf/iotdb-datanode.properties | 2 +- .../test/resources/datanode1conf/iotdb-metric.yml | 2 +- .../datanode2conf/iotdb-datanode.properties | 3 +- .../test/resources/datanode2conf/iotdb-metric.yml | 2 +- .../test/resources/datanode3conf/iotdb-metric.yml | 2 +- server/src/test/resources/logback-test.xml | 2 +- site/src/main/.vuepress/config.js | 2 + .../apache/iotdb/spark/db/EnvironmentUtils.java | 4 +- thrift-commons/src/main/thrift/common.thrift | 1 - .../src/main/thrift/confignode.thrift | 50 ++- thrift/src/main/thrift/datanode.thrift | 19 +- 285 files changed, 8763 insertions(+), 5789 deletions(-)
