This is an automated email from the ASF dual-hosted git repository. hui pushed a commit to branch QueryMetrics in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit d209479edaef4df6d02a4cf60a5bc1bb2ec68121 Merge: 5d654bdbbd d6671b3662 Author: Minghui Liu <[email protected]> AuthorDate: Tue Dec 6 22:24:56 2022 +0800 Merge remote-tracking branch 'origin/master' into QueryMetrics # Conflicts: # server/src/main/java/org/apache/iotdb/db/mpp/execution/fragment/FragmentInstanceManager.java .github/workflows/main-unix.yml | 2 +- README.md | 16 +- README_ZH.md | 14 +- .../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 5 +- .../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 4 + client-py/README.md | 7 +- client-py/requirements_dev.txt | 2 +- confignode/pom.xml | 2 +- .../src/assembly/resources/conf/confignode-env.bat | 2 + .../src/assembly/resources/conf/confignode-env.sh | 2 +- .../iotdb/confignode/conf/ConfigNodeConfig.java | 32 +- .../confignode/conf/ConfigNodeDescriptor.java | 52 +- .../request/read/region/GetRegionIdPlan.java | 2 +- .../iotdb/confignode/manager/ConfigManager.java | 9 +- .../iotdb/confignode/manager/ConsensusManager.java | 5 +- .../apache/iotdb/confignode/manager/IManager.java | 4 +- .../iotdb/confignode/manager/node/NodeManager.java | 6 +- .../manager/partition/PartitionManager.java | 16 +- .../persistence/executor/ConfigPlanExecutor.java | 14 +- .../partition/StorageGroupPartitionTable.java | 2 + .../iotdb/confignode/service/ConfigNode.java | 4 + .../confignode/service/ConfigNodeCommandLine.java | 3 +- .../thrift/ConfigNodeRPCServiceProcessor.java | 11 +- consensus/pom.xml | 2 +- .../iotdb/consensus/ratis/RatisConsensus.java | 11 +- .../iotdb/consensus/ratis/RatisConsensusTest.java | 5 + cross-tests/pom.xml | 2 +- docs/Download/README.md | 46 +- docs/UserGuide/Cluster/Cluster-Concept.md | 5 +- .../Cluster-Maintenance.md} | 274 +-- docs/UserGuide/Cluster/Cluster-Setup.md | 13 +- .../UserGuide/Cluster/Deployment-Recommendation.md | 181 ++ docs/UserGuide/Data-Concept/Data-Type.md | 2 +- .../Maintenance-Tools/Maintenance-Command.md | 383 ---- .../Operators-Functions/Data-Profiling.md | 4 +- docs/UserGuide/Operators-Functions/String.md | 4 +- .../Operators-Functions/User-Defined-Function.md | 2 +- .../UserGuide/QuickStart/Command-Line-Interface.md | 7 +- docs/UserGuide/QuickStart/QuickStart.md | 15 +- docs/UserGuide/Reference/Common-Config-Manual.md | 23 +- docs/UserGuide/UDF-Library/M4.md | 26 + docs/zh/Download/README.md | 54 +- docs/zh/UserGuide/Cluster/Cluster-Concept.md | 4 +- .../Cluster-Maintenance.md} | 223 +-- docs/zh/UserGuide/Cluster/Cluster-Setup.md | 13 +- .../UserGuide/Cluster/Deployment-Recommendation.md | 179 ++ docs/zh/UserGuide/Data-Concept/Data-Type.md | 2 +- .../Maintenance-Tools/Maintenance-Command.md | 363 ---- .../Operators-Functions/Data-Profiling.md | 4 +- docs/zh/UserGuide/Operators-Functions/Overview.md | 2 +- docs/zh/UserGuide/Operators-Functions/String.md | 4 +- .../Operators-Functions/User-Defined-Function.md | 2 +- .../UserGuide/QuickStart/Command-Line-Interface.md | 4 +- docs/zh/UserGuide/QuickStart/QuickStart.md | 14 +- .../zh/UserGuide/Reference/Common-Config-Manual.md | 22 +- .../Reference/ConfigNode-Config-Manual.md | 2 +- integration-test/pom.xml | 212 +- .../java/org/apache/iotdb/it/env/MppConfig.java | 9 + .../org/apache/iotdb/itbase/env/BaseConfig.java | 8 + .../IoTDBCustomRegionGroupExtensionIT.java | 39 +- .../it/partition/IoTDBPartitionGetterIT.java | 4 +- .../it/aggregation/IoTDBAggregationOptimizeIT.java | 181 ++ .../iotdb/db/it/env/StandaloneEnvConfig.java | 14 + .../iotdb/db/it/selectinto/IoTDBSelectInto2IT.java | 3 + .../iotdb/db/it/selectinto/IoTDBSelectInto3IT.java | 3 + .../iotdb/db/it/selectinto/IoTDBSelectIntoIT.java | 5 + .../iotdb/session/it/IoTDBSessionSimpleIT.java | 31 + integration/pom.xml | 20 +- .../db/integration/IoTDBLoadExternalTsfileIT.java | 1032 ---------- .../IoTDBLoadExternalTsfileWithVirtualSGIT.java | 137 -- .../db/integration/IoTDBNewTsFileCompactionIT.java | 36 +- .../db/integration/IoTDBRewriteTsFileToolIT.java | 59 +- .../iotdb/db/integration/IoTDBSnapshotIT.java | 285 --- .../aligned/IoTDBLoadExternalAlignedTsFileIT.java | 2027 ++++++++++---------- .../resources/conf/iotdb-common.properties | 13 +- .../assembly/resources/sbin/start-standalone.bat | 3 + .../assembly/resources/sbin/stop-standalone.bat | 5 +- .../commons/auth/role/LocalFileRoleAccessor.java | 23 +- .../apache/iotdb/commons/conf/IoTDBConstant.java | 8 +- .../commons/partition/DataPartitionTable.java | 6 +- .../commons/partition/SchemaPartitionTable.java | 6 +- .../commons/partition/SeriesPartitionTable.java | 12 +- .../org/apache/iotdb/commons/path/AlignedPath.java | 18 + .../org/apache/iotdb/commons/path/PartialPath.java | 3 + .../iotdb/commons/service/StartupChecks.java | 44 +- pom.xml | 6 +- server/pom.xml | 2 +- .../src/assembly/resources/conf/datanode-env.bat | 2 + server/src/assembly/resources/conf/datanode-env.sh | 2 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 43 +- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 22 +- .../db/consensus/DataRegionConsensusImpl.java | 7 +- .../db/consensus/SchemaRegionConsensusImpl.java | 3 +- .../statemachine/DataRegionStateMachine.java | 16 +- .../org/apache/iotdb/db/engine/StorageEngine.java | 777 +++++++- .../apache/iotdb/db/engine/StorageEngineV2.java | 820 -------- .../db/engine/compaction/CompactionUtils.java | 40 + .../compaction/cross/CrossSpaceCompactionTask.java | 12 + .../rewrite/CrossSpaceCompactionResource.java | 13 +- .../RewriteCrossSpaceCompactionSelector.java | 5 +- .../compaction/inner/InnerSpaceCompactionTask.java | 11 + .../writer/AbstractCompactionWriter.java | 6 +- .../writer/AbstractCrossCompactionWriter.java | 2 +- .../iotdb/db/engine/load/LoadTsFileManager.java | 2 +- .../iotdb/db/engine/snapshot/SnapshotLoader.java | 4 +- .../iotdb/db/engine/snapshot/SnapshotTaker.java | 8 +- .../iotdb/db/engine/storagegroup/DataRegion.java | 173 +- .../db/engine/storagegroup/DataRegionMetrics.java | 2 +- .../engine/storagegroup/IDataRegionForQuery.java | 47 + .../engine/storagegroup/TimePartitionManager.java | 4 +- .../db/engine/storagegroup/TsFileProcessor.java | 4 +- .../engine/storagegroup/TsFileProcessorInfo.java | 2 +- .../db/engine/storagegroup/TsFileResourceList.java | 4 +- .../db/engine/storagegroup/VirtualDataRegion.java | 78 + .../storagegroup/timeindex/DeviceTimeIndex.java | 9 +- .../storagegroup/timeindex/FileTimeIndex.java | 6 +- .../iotdb/db/localconfignode/LocalConfigNode.java | 4 +- .../db/mpp/aggregation/ExtremeAccumulator.java | 20 +- .../db/mpp/aggregation/FirstValueAccumulator.java | 28 +- .../db/mpp/aggregation/LastValueAccumulator.java | 27 +- .../db/mpp/aggregation/MaxValueAccumulator.java | 21 +- .../db/mpp/aggregation/MinTimeAccumulator.java | 1 + .../db/mpp/aggregation/MinValueAccumulator.java | 21 +- .../iotdb/db/mpp/aggregation/SumAccumulator.java | 1 + .../db/mpp/common/header/ColumnHeaderConstant.java | 12 +- .../iotdb/db/mpp/execution/driver/DataDriver.java | 4 +- .../db/mpp/execution/driver/DataDriverContext.java | 8 +- .../fragment/FragmentInstanceContext.java | 22 +- .../fragment/FragmentInstanceManager.java | 21 +- .../operator/process/AbstractIntoOperator.java | 215 ++- .../operator/process/DeviceViewIntoOperator.java | 105 +- .../execution/operator/process/IntoOperator.java | 48 +- .../process/join/VerticallyConcatOperator.java | 207 ++ ...r.java => AbstractUpdateLastCacheOperator.java} | 74 +- .../last/AlignedUpdateLastCacheOperator.java | 89 + .../operator/process/last/LastQueryOperator.java | 6 +- .../process/last/LastQuerySortOperator.java | 11 +- .../operator/process/last/LastQueryUtil.java | 18 + .../process/last/UpdateLastCacheOperator.java | 84 +- .../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 1 - .../plan/analyze/StandalonePartitionFetcher.java | 4 +- .../config/executor/ClusterConfigTaskExecutor.java | 11 +- .../config/metadata/GetTimeSlotListTask.java | 7 +- .../memory/StatementMemorySourceVisitor.java | 10 +- .../iotdb/db/mpp/plan/parser/ASTVisitor.java | 17 +- .../db/mpp/plan/parser/StatementGenerator.java | 16 +- .../plan/planner/LocalExecutionPlanContext.java | 5 + .../db/mpp/plan/planner/LocalExecutionPlanner.java | 4 +- .../db/mpp/plan/planner/LogicalPlanBuilder.java | 20 +- .../db/mpp/plan/planner/OperatorTreeGenerator.java | 169 +- .../db/mpp/plan/planner/SubPlanTypeExtractor.java | 29 + .../planner/distribution/ExchangeNodeAdder.java | 7 + .../plan/planner/distribution/SourceRewriter.java | 248 ++- .../plan/planner/plan/node/PlanGraphPrinter.java | 8 + .../mpp/plan/planner/plan/node/PlanNodeType.java | 6 +- .../db/mpp/plan/planner/plan/node/PlanVisitor.java | 5 + .../plan/node/process/VerticallyConcatNode.java | 103 + .../planner/plan/parameter/AggregationStep.java | 45 +- .../db/mpp/plan/scheduler/StandaloneScheduler.java | 7 +- .../scheduler/load/LoadTsFileDispatcherImpl.java | 9 +- .../statement/metadata/GetRegionIdStatement.java | 30 +- .../db/query/control/QueryResourceManager.java | 38 +- .../groupby/GroupByWithValueFilterDataSet.java | 88 +- .../groupby/GroupByWithoutValueFilterDataSet.java | 106 +- .../db/query/executor/AggregationExecutor.java | 124 +- .../iotdb/db/query/executor/FillQueryExecutor.java | 69 +- .../iotdb/db/query/executor/LastQueryExecutor.java | 50 +- .../db/query/executor/RawDataQueryExecutor.java | 138 +- .../query/timegenerator/ServerTimeGenerator.java | 28 +- .../org/apache/iotdb/db/rescon/SystemInfo.java | 10 +- .../java/org/apache/iotdb/db/service/DataNode.java | 21 +- .../java/org/apache/iotdb/db/service/IoTDB.java | 5 +- .../apache/iotdb/db/service/IoTDBShutdownHook.java | 4 +- .../java/org/apache/iotdb/db/service/NewIoTDB.java | 8 +- .../iotdb/db/service/RegionMigrateService.java | 4 +- .../impl/DataNodeInternalRPCServiceImpl.java | 8 +- .../service/thrift/impl/DataNodeRegionManager.java | 4 +- .../db/sync/sender/manager/LocalSyncManager.java | 2 +- .../iotdb/db/sync/sender/pipe/TsFilePipe.java | 8 +- .../sync/transport/client/SyncClientFactory.java | 7 +- .../iotdb/db/tools/TsFileSplitByPartitionTool.java | 10 +- .../db/tools/upgrade/TsFileOnlineUpgradeTool.java | 6 +- .../org/apache/iotdb/db/utils/ThreadUtils.java | 3 +- .../java/org/apache/iotdb/db/wal/node/WALNode.java | 4 +- ...ageEngineV2Test.java => StorageEngineTest.java} | 14 +- ...CrossSpaceCompactionWithFastPerformerTest.java} | 19 +- ...CompactionWithFastPerformerValidationTest.java} | 201 +- ...SpaceCompactionWithReadPointPerformerTest.java} | 19 +- ...ctionWithReadPointPerformerValidationTest.java} | 253 +-- ...CrossSpaceCompactionWithFastPerformerTest.java} | 2 +- ...SpaceCompactionWithReadPointPerformerTest.java} | 12 +- .../utils/MultiTsFileDeviceIteratorTest.java | 75 + .../db/engine/snapshot/IoTDBSnapshotTest.java | 2 +- .../db/engine/storagegroup/DataRegionTest.java | 22 +- ...ocessorV2Test.java => TsFileProcessorTest.java} | 6 +- .../mpp/execution/operator/OperatorMemoryTest.java | 5 +- .../operator/VerticallyConcatOperatorTest.java | 197 ++ .../db/mpp/plan/StandaloneCoordinatorTest.java | 6 +- .../distribution/AggregationDistributionTest.java | 70 +- .../plan/scheduler/StandaloneSchedulerTest.java | 6 +- .../iotdb/db/tools/TsFileAndModSettleToolTest.java | 373 ++-- .../apache/iotdb/db/utils/EnvironmentUtils.java | 4 +- site/iotdb-doap.rdf | 8 +- site/src/main/.vuepress/config.js | 29 +- site/src/main/.vuepress/redirects | 3 +- site/src/main/.vuepress/theme/components/Page.vue | 36 +- site/src/main/package.json | 9 +- .../apache/iotdb/spark/db/EnvironmentUtils.java | 7 +- testcontainer/pom.xml | 2 +- .../src/main/thrift/confignode.thrift | 6 +- tsfile/pom.xml | 2 +- 211 files changed, 5704 insertions(+), 6728 deletions(-) diff --cc node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java index b5a31a5a95,2e58d01d60..45de4c9903 --- a/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java +++ b/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java @@@ -78,12 -81,10 +81,13 @@@ public class IoTDBConstant public static final String AUDIT_LOGGER_NAME = "IoTDB_AUDIT_LOGGER"; public static final String SLOW_SQL_LOGGER_NAME = "SLOW_SQL"; + + public static final String QUERY_STATISTICS_LOGGER_NAME = "QUERY_STATISTICS"; + public static final String COMPACTION_LOGGER_NAME = "COMPACTION"; - public static final String IOTDB_JMX_PORT = "iotdb.jmx.port"; + public static final String IOTDB_JMX_LOCAL = "iotdb.jmx.local"; + public static final String IOTDB_JMX_PORT = "com.sun.management.jmxremote.port"; public static final String IOTDB_PACKAGE = "org.apache.iotdb.service"; public static final String IOTDB_THREADPOOL_PACKAGE = "org.apache.iotdb.threadpool"; diff --cc server/src/main/java/org/apache/iotdb/db/mpp/execution/fragment/FragmentInstanceManager.java index d9d0aa6b1f,5e37ac29aa..a502da89f0 --- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/fragment/FragmentInstanceManager.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/fragment/FragmentInstanceManager.java @@@ -70,8 -68,8 +70,10 @@@ public class FragmentInstanceManager private static final long QUERY_TIMEOUT_MS = IoTDBDescriptor.getInstance().getConfig().getQueryTimeoutThreshold(); + private static final QueryStatistics QUERY_STATISTICS = QueryStatistics.getInstance(); + + private final ExecutorService intoOperationExecutor; + public static FragmentInstanceManager getInstance() { return FragmentInstanceManager.InstanceHolder.INSTANCE; } @@@ -97,10 -100,9 +104,10 @@@ } public FragmentInstanceInfo execDataQueryFragmentInstance( - FragmentInstance instance, DataRegion dataRegion) { + FragmentInstance instance, IDataRegionForQuery dataRegion) { FragmentInstanceId instanceId = instance.getId(); + long startTime = System.nanoTime(); try (SetThreadName fragmentInstanceName = new SetThreadName(instanceId.getFullId())) { FragmentInstanceExecution execution = instanceExecution.computeIfAbsent(
