This is an automated email from the ASF dual-hosted git repository. xuekaifeng pushed a commit to branch xkf_fix_idtable_compaction in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 0b997f1c7bb9fce2aac735a1dfbb4f83a2454a05 Merge: a75cf65 3deadae Author: 151250176 <[email protected]> AuthorDate: Thu Mar 17 15:02:55 2022 +0800 Merge branch 'master' of https://github.com/apache/iotdb into xkf_fix_idtable_compaction .github/dependabot.yml | 26 + README.md | 4 +- README_ZH.md | 4 +- .../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 8 +- .../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 11 +- .../org/apache/iotdb/cluster/ClusterIoTDB.java | 51 +- .../cluster/ClusterIoTDBServerCommandLine.java | 94 ++ .../cluster/client/async/AsyncDataClient.java | 2 +- .../cluster/client/async/AsyncMetaClient.java | 2 +- .../iotdb/cluster/client/sync/SyncDataClient.java | 2 +- .../iotdb/cluster/client/sync/SyncMetaClient.java | 2 +- .../iotdb/cluster/config/ClusterConstant.java | 2 +- .../iotdb/cluster/coordinator/Coordinator.java | 2 +- .../apache/iotdb/cluster/log/LogDispatcher.java | 2 +- .../iotdb/cluster/log/applier/BaseApplier.java | 2 +- .../iotdb/cluster/log/catchup/CatchUpTask.java | 2 +- .../iotdb/cluster/log/catchup/LogCatchUpTask.java | 2 +- .../cluster/log/manage/CommittedEntryManager.java | 2 +- .../iotdb/cluster/log/manage/RaftLogManager.java | 2 +- .../log/manage/UnCommittedEntryManager.java | 2 +- .../serializable/SyncLogDequeSerializer.java | 2 +- .../apache/iotdb/cluster/metadata/CMManager.java | 9 +- .../iotdb/cluster/query/ClusterPlanExecutor.java | 2 +- .../iotdb/cluster/query/ClusterPlanRouter.java | 2 +- .../iotdb/cluster/query/LocalQueryExecutor.java | 4 +- .../groupby/ClusterGroupByNoVFilterDataSet.java | 4 +- .../groupby/ClusterGroupByVFilterDataSet.java | 2 +- .../query/groupby/MergeGroupByExecutor.java | 2 +- .../query/groupby/RemoteGroupByExecutor.java | 2 +- .../cluster/query/manage/ClusterQueryManager.java | 2 +- .../cluster/query/reader/ClusterReaderFactory.java | 2 +- .../cluster/query/reader/ClusterTimeGenerator.java | 2 +- .../iotdb/cluster/query/reader/EmptyReader.java | 2 +- .../cluster/server/member/DataGroupMember.java | 2 +- .../cluster/server/member/MetaGroupMember.java | 2 +- .../iotdb/cluster/server/member/RaftMember.java | 2 +- .../cluster/server/monitor/NodeStatusManager.java | 2 +- .../cluster/server/service/DataGroupEngine.java | 2 +- .../apache/iotdb/cluster/utils/PartitionUtils.java | 2 - .../cluster/log/applier/MetaLogApplierTest.java | 10 - .../query/groupby/RemoteGroupByExecutorTest.java | 2 +- code-coverage/pom.xml | 2 +- confignode/pom.xml | 17 +- .../src/assembly/resources/conf/confignode-env.bat | 146 ++ .../src/assembly/resources/conf/confignode-env.sh | 264 ++++ .../resources/conf/iotdb-confignode.properties | 10 +- .../{iotdb-confignode.properties => jmx.access} | 21 +- .../{iotdb-confignode.properties => jmx.password} | 21 +- .../assembly/resources/sbin/start-confignode.bat | 121 ++ .../assembly/resources/sbin/start-confignode.sh | 77 + .../assembly/resources/sbin/stop-confignode.bat | 38 + .../resources/sbin/stop-confignode.sh} | 32 +- .../iotdb/confignode/conf/ConfigNodeConf.java | 5 +- .../iotdb/confignode/conf/ConfigNodeConfCheck.java | 136 ++ .../iotdb/confignode/conf/ConfigNodeConstant.java | 28 +- .../confignode/conf/ConfigNodeDescriptor.java | 66 +- .../confignode/exception/ConfigNodeException.java | 11 +- .../conf/RepeatConfigurationException.java} | 18 +- .../startup/StartupException.java} | 17 +- .../iotdb/confignode/manager/ConfigManager.java | 4 +- .../iotdb/confignode/service/ConfigNode.java | 112 ++ .../confignode/service/ConfigNodeCommandLine.java | 80 + .../iotdb/confignode/service/ConfigNodeMBean.java | 10 +- .../register/IService.java} | 39 +- .../confignode/service/register/JMXService.java | 105 ++ .../service/register}/RegisterManager.java | 33 +- .../register/ServiceType.java} | 30 +- .../confignode/service/startup/StartupCheck.java | 11 +- .../confignode/service/startup/StartupChecks.java | 89 ++ .../{impl => server}/ConfigNodeRPCServer.java | 2 +- .../confignode/conf/ConfigNodeDescriptorTest.java | 76 - .../manager/hash/DeviceGroupHashExecutorTest.java | 3 +- .../utils/ConfigNodeEnvironmentUtils.java | 133 ++ docs/Download/README.md | 16 +- .../Administration-Management/Administration.md | 6 +- .../Maintenance-Tools/Maintenance-Command.md | 8 - docs/UserGuide/Operate-Metadata/Timeseries.md | 41 +- docs/UserGuide/Query-Data/Aggregate-Query.md | 28 +- docs/UserGuide/Query-Data/Overview.md | 2 +- docs/UserGuide/Reference/Config-Manual.md | 7 +- docs/UserGuide/Reference/SQL-Reference.md | 5 - docs/zh/Download/README.md | 11 +- .../Administration-Management/Administration.md | 8 +- .../Maintenance-Tools/Maintenance-Command.md | 7 - docs/zh/UserGuide/Operate-Metadata/Timeseries.md | 36 +- docs/zh/UserGuide/Query-Data/Aggregate-Query.md | 37 +- docs/zh/UserGuide/Query-Data/Overview.md | 2 +- docs/zh/UserGuide/Reference/Config-Manual.md | 5 +- docs/zh/UserGuide/Reference/SQL-Reference.md | 6 - .../iotdb/AlignedTimeseriesSessionExample.java | 9 +- .../main/java/org/apache/iotdb/SessionExample.java | 5 +- .../apache/iotdb/db/integration/IOTDBInsertIT.java | 26 + .../iotdb/db/integration/IoTDBAuthorizationIT.java | 49 + .../db/integration/IoTDBCreateSnapshotIT.java | 180 --- .../iotdb/db/integration/IoTDBMetadataFetchIT.java | 43 +- .../db/integration/IoTDBResultMetadataIT.java | 82 + .../IoTDBGroupByQueryWithoutValueFilterIT.java | 14 - ...roupBySlidingWindowQueryWithValueFilterIT.java} | 575 +++----- ...pBySlidingWindowQueryWithoutValueFilterIT.java} | 552 +++---- .../integration/{ => groupby}/IOTDBGroupByIT.java | 2 +- .../{ => groupby}/IOTDBGroupByInnerIntervalIT.java | 2 +- .../{ => groupby}/IoTDBGroupByFillIT.java | 2 +- .../{ => groupby}/IoTDBGroupByFillMixPathsIT.java | 2 +- .../{ => groupby}/IoTDBGroupByFillWithRangeIT.java | 2 +- .../{ => groupby}/IoTDBGroupByMonthFillIT.java | 2 +- .../{ => groupby}/IoTDBGroupByMonthIT.java | 95 +- .../{ => groupby}/IoTDBGroupByUnseqIT.java | 2 +- .../apache/iotdb/session/IoTDBSessionSimpleIT.java | 5 +- iotdb-commons/pom.xml | 159 ++ .../apache/iotdb/commons/ServerCommandLine.java | 67 + .../apache/iotdb/commons}/hash/APHashExecutor.java | 2 +- .../iotdb/commons}/hash/BKDRHashExecutor.java | 2 +- .../commons}/hash/DeviceGroupHashExecutor.java | 2 +- .../apache/iotdb/commons}/hash/JSHashExecutor.java | 2 +- .../iotdb/commons}/hash/SDBMHashExecutor.java | 2 +- .../org/apache/iotdb/commons}/utils/TestOnly.java | 2 +- .../org/apache/iotdb/jdbc/IoTDBResultMetadata.java | 29 +- pom.xml | 14 +- server/pom.xml | 9 + .../resources/conf/iotdb-engine.properties | 36 +- .../org/apache/iotdb/db/auth/AuthorityChecker.java | 2 + .../apache/iotdb/db/auth/entity/PrivilegeType.java | 38 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 36 - .../org/apache/iotdb/db/conf/IoTDBConfigCheck.java | 4 +- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 40 +- .../org/apache/iotdb/db/engine/StorageEngine.java | 2 +- .../iotdb/db/engine/cache/BloomFilterCache.java | 2 +- .../apache/iotdb/db/engine/cache/ChunkCache.java | 2 +- .../db/engine/cache/TimeSeriesMetadataCache.java | 2 +- .../engine/compaction/CompactionTaskManager.java | 2 +- .../utils/SingleSeriesCompactionExecutor.java | 9 +- .../iotdb/db/engine/cq/ContinuousQueryService.java | 2 +- .../db/engine/storagegroup/StorageGroupInfo.java | 2 +- .../db/engine/storagegroup/TsFileProcessor.java | 2 +- .../db/engine/storagegroup/TsFileResource.java | 2 +- .../db/engine/storagegroup/TsFileResourceList.java | 2 +- .../storagegroup/VirtualStorageGroupProcessor.java | 2 +- .../engine/trigger/executor/TriggerExecutor.java | 2 +- .../service/TriggerRegistrationService.java | 2 +- .../SchemaDirCreationFailureException.java | 12 +- .../org/apache/iotdb/db/metadata/MManager.java | 1559 ++++++-------------- .../apache/iotdb/db/metadata/MetadataConstant.java | 3 + .../idtable/AppendOnlyDiskSchemaManager.java | 2 +- .../apache/iotdb/db/metadata/idtable/IDTable.java | 2 +- .../db/metadata/idtable/IDTableHashmapImpl.java | 2 +- .../db/metadata/idtable/IDiskSchemaManager.java | 2 +- .../db/metadata/idtable/entry/DeviceEntry.java | 2 +- .../db/metadata/idtable/entry/DeviceIDFactory.java | 2 +- .../idtable/entry/InsertMeasurementMNode.java | 7 +- .../db/metadata/idtable/entry/SchemaEntry.java | 2 +- .../iotdb/db/metadata/logfile/MLogUpgrader.java | 290 ---- .../iotdb/db/metadata/mnode/EntityMNode.java | 13 + .../org/apache/iotdb/db/metadata/mnode/IMNode.java | 7 +- .../db/metadata/mnode/IStorageGroupMNode.java | 6 + .../iotdb/db/metadata/mnode/InternalMNode.java | 38 +- .../org/apache/iotdb/db/metadata/mnode/MNode.java | 14 +- .../iotdb/db/metadata/mnode/MeasurementMNode.java | 3 +- .../db/metadata/mnode/StorageGroupEntityMNode.java | 23 + .../iotdb/db/metadata/mnode/StorageGroupMNode.java | 23 + .../iotdb/db/metadata/mtree/MTreeAboveSG.java | 559 +++++++ .../mtree/{MTree.java => MTreeBelowSG.java} | 1105 +++----------- .../db/metadata/mtree/traverser/Traverser.java | 86 +- .../MNodeAboveSGCollector.java} | 48 +- .../traverser/collector/MeasurementCollector.java | 20 +- ...lCounter.java => MNodeAboveSGLevelCounter.java} | 47 +- .../mtree/traverser/counter/MNodeLevelCounter.java | 10 +- .../counter/MeasurementGroupByLevelCounter.java | 26 + .../apache/iotdb/db/metadata/path/AlignedPath.java | 2 +- .../iotdb/db/metadata/path/MeasurementPath.java | 2 +- .../apache/iotdb/db/metadata/path/PartialPath.java | 2 +- .../db/metadata/rescon/TimeseriesStatistics.java | 104 ++ .../storagegroup/IStorageGroupSchemaManager.java | 232 +++ .../SGMManager.java} | 1255 ++++------------ .../storagegroup/StorageGroupSchemaManager.java | 252 ++++ .../apache/iotdb/db/metadata/tag/TagManager.java | 27 +- .../template/TemplateLogReader.java} | 34 +- .../db/metadata/template/TemplateLogWriter.java | 64 + .../db/metadata/template/TemplateManager.java | 123 +- .../db/metadata/upgrade/MetadataUpgrader.java | 425 ++++++ .../apache/iotdb/db/metadata/utils/MetaUtils.java | 2 +- .../org/apache/iotdb/db/mpp/buffer/SinkHandle.java | 56 + .../apache/iotdb/db/mpp/buffer/SourceHandle.java | 22 +- .../common/Analysis.java} | 13 +- .../common/FillPolicy.java} | 12 +- .../common/GroupByTimeParameter.java} | 16 +- .../common/OrderBy.java} | 16 +- .../common/QueryContext.java} | 18 +- .../common/QueryId.java} | 16 +- .../common/QuerySession.java} | 12 +- .../org/apache/iotdb/db/mpp/common/TreeNode.java | 25 +- .../org/apache/iotdb/db/mpp/common/TsBlock.java | 48 + .../iotdb/db/mpp/common/TsBlockMetadata.java | 41 + .../common/WithoutPolicy.java} | 13 +- .../execution/Coordinator.java} | 32 +- .../db/mpp/execution/ExecFragmentInstance.java | 57 + .../iotdb/db/mpp/execution/QueryExecution.java | 96 ++ .../iotdb/db/mpp/execution/QueryScheduler.java | 51 + .../execution/QueryStateMachine.java} | 14 +- .../org/apache/iotdb/db/mpp/operator/Operator.java | 49 + .../operator/OperatorContext.java} | 31 +- .../iotdb/db/mpp/operator/SeriesScanOperator.java | 21 +- .../iotdb/db/mpp/plan/DistributedQueryPlan.java | 20 +- .../iotdb/db/mpp/plan/DistributionPlanner.java | 21 +- .../apache/iotdb/db/mpp/plan/FragmentInstance.java | 17 +- .../plan/FragmentInstanceId.java} | 15 +- .../apache/iotdb/db/mpp/plan/LogicalPlanner.java | 35 +- .../apache/iotdb/db/mpp/plan/LogicalQueryPlan.java | 22 +- .../org/apache/iotdb/db/mpp/plan/PlanFragment.java | 19 +- .../plan/PlanFragmentId.java} | 14 +- .../TestOnly.java => mpp/plan/node/PlanNode.java} | 26 +- .../plan/node/PlanNodeId.java} | 21 +- .../plan/node/PlanNodeIdAllocator.java} | 13 +- .../db/mpp/plan/node/process/DeviceMergeNode.java | 66 + .../iotdb/db/mpp/plan/node/process/FillNode.java | 30 +- .../iotdb/db/mpp/plan/node/process/FilterNode.java | 30 +- .../db/mpp/plan/node/process/FilterNullNode.java | 30 +- .../db/mpp/plan/node/process/GroupByLevelNode.java | 41 + .../iotdb/db/mpp/plan/node/process/LimitNode.java | 23 +- .../iotdb/db/mpp/plan/node/process/OffsetNode.java | 26 +- .../db/mpp/plan/node/process/ProcessNode.java | 17 +- .../node/process/RowBasedSeriesAggregateNode.java | 59 + .../iotdb/db/mpp/plan/node/process/SortNode.java | 32 +- .../db/mpp/plan/node/process/TimeJoinNode.java | 67 + .../iotdb/db/mpp/plan/node/sink/CsvSinkNode.java | 15 +- .../db/mpp/plan/node/sink/FragmentSinkNode.java | 17 +- .../iotdb/db/mpp/plan/node/sink/SinkNode.java | 16 +- .../db/mpp/plan/node/sink/ThriftSinkNode.java | 17 +- .../db/mpp/plan/node/source/CsvSourceNode.java | 17 +- .../mpp/plan/node/source/SeriesAggregateNode.java | 81 + .../db/mpp/plan/node/source/SeriesScanNode.java | 85 ++ .../iotdb/db/mpp/plan/node/source/SourceNode.java | 16 +- .../plan/optimzation/PlanOptimizer.java} | 14 +- .../apache/iotdb/db/qp/executor/PlanExecutor.java | 7 - .../sys/CreateAlignedTimeSeriesOperator.java | 57 +- .../apache/iotdb/db/qp/physical/PhysicalPlan.java | 6 +- .../iotdb/db/qp/physical/crud/AggregationPlan.java | 2 +- .../db/qp/physical/crud/GroupByTimeFillPlan.java | 7 +- .../iotdb/db/qp/physical/crud/InsertRowPlan.java | 2 +- .../physical/sys/CreateAlignedTimeSeriesPlan.java | 179 ++- .../db/qp/physical/sys/CreateSnapshotPlan.java | 56 - .../db/qp/physical/sys/CreateTemplatePlan.java | 2 +- .../apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java | 28 +- .../iotdb/db/qp/strategy/LogicalGenerator.java | 2 +- .../apache/iotdb/db/qp/utils/DatetimeUtils.java | 24 +- .../db/query/aggregation/AggregateResult.java | 53 +- .../aggregation/RemovableAggregateResult.java} | 13 +- .../db/query/aggregation/impl/AvgAggrResult.java | 22 +- .../db/query/aggregation/impl/CountAggrResult.java | 14 +- .../query/aggregation/impl/ExtremeAggrResult.java | 8 +- .../aggregation/impl/FirstValueAggrResult.java | 3 - .../aggregation/impl/LastValueAggrResult.java | 3 - .../query/aggregation/impl/MaxTimeAggrResult.java | 5 + .../query/aggregation/impl/MaxValueAggrResult.java | 4 + .../query/aggregation/impl/MinTimeAggrResult.java | 5 + .../query/aggregation/impl/MinValueAggrResult.java | 4 + .../db/query/aggregation/impl/SumAggrResult.java | 20 +- .../dataset/groupby/GroupByEngineDataSet.java | 239 --- .../query/dataset/groupby/GroupByFillDataSet.java | 2 +- .../query/dataset/groupby/GroupByLevelDataSet.java | 2 +- .../query/dataset/groupby/GroupByTimeDataSet.java | 177 +++ .../dataset/groupby/GroupByTimeEngineDataSet.java | 91 ++ .../groupby/GroupByWithValueFilterDataSet.java | 87 +- .../groupby/GroupByWithoutValueFilterDataSet.java | 104 +- .../iotdb/db/query/executor/QueryRouter.java | 10 +- .../apache/iotdb/db/query/executor/fill/IFill.java | 13 +- .../groupby/AlignedGroupByExecutor.java | 2 +- .../groupby/GroupByExecutor.java | 2 +- .../groupby/SlidingWindowGroupByExecutor.java | 66 + .../SlidingWindowGroupByExecutorFactory.java | 117 ++ .../EmptyQueueSlidingWindowGroupByExecutor.java | 51 + .../groupby/impl}/LocalAlignedGroupByExecutor.java | 3 +- .../groupby/impl}/LocalGroupByExecutor.java | 3 +- ...MonotonicQueueSlidingWindowGroupByExecutor.java | 73 + .../NormalQueueSlidingWindowGroupByExecutor.java | 61 + .../SmoothQueueSlidingWindowGroupByExecutor.java | 55 + .../db/query/factory/AggregateResultFactory.java | 2 + .../query/reader/series/AlignedSeriesReader.java | 2 +- .../query/reader/series/SeriesAggregateReader.java | 2 +- .../reader/series/SeriesRawDataBatchReader.java | 2 +- .../iotdb/db/query/reader/series/SeriesReader.java | 2 +- .../reader/series/SeriesReaderByTimestamp.java | 2 +- .../query/udf/service/UDFRegistrationService.java | 2 +- .../iotdb/db/rescon/TsFileResourceManager.java | 2 +- .../apache/iotdb/db/service/RegisterManager.java | 2 +- .../db/service/thrift/impl/TSServiceImpl.java | 4 +- .../db/tools/virtualsg/DeviceMappingViewer.java | 1 - .../java/org/apache/iotdb/db/utils/AuthUtils.java | 2 + .../org/apache/iotdb/db/utils/CommonUtils.java | 13 +- .../db/utils/datastructure/AlignedTVList.java | 2 +- .../iotdb/db/utils/datastructure/TVList.java | 2 +- .../iotdb/db/utils/datastructure/TimeSelector.java | 18 + .../timerangeiterator/AggrWindowIterator.java | 149 ++ .../timerangeiterator/ITimeRangeIterator.java | 27 +- .../timerangeiterator/PreAggrWindowIterator.java | 142 ++ .../PreAggrWindowWithNaturalMonthIterator.java | 109 ++ .../TimeRangeIteratorFactory.java | 69 + .../utils/windowing/window/EvictableBatchList.java | 2 +- .../org/apache/iotdb/db/writelog/io/LogWriter.java | 2 +- .../InnerSpaceCompactionUtilsNoAlignedTest.java | 83 +- .../iotdb/db/metadata/MManagerBasicTest.java | 124 +- .../org/apache/iotdb/db/metadata/MTreeTest.java | 1060 ------------- .../iotdb/db/metadata/idtable/IDTableTest.java | 4 + .../iotdb/db/metadata/mlog/MLogUpgraderTest.java | 176 --- .../iotdb/db/metadata/mtree/MTreeAboveSGTest.java | 292 ++++ .../iotdb/db/metadata/mtree/MTreeBelowSGTest.java | 796 ++++++++++ .../db/metadata/upgrade/MetadataUpgradeTest.java | 304 ++++ .../db/qp/physical/PhysicalPlanSerializeTest.java | 2 + ...imeDataSetTest.java => GroupByDataSetTest.java} | 4 +- .../dataset/groupby/GroupByEngineDataSetTest.java | 504 ------- .../dataset/groupby/GroupByTimeDataSetTest.java | 636 +++++--- .../org/apache/iotdb/db/tools/MLogParserTest.java | 88 +- .../iotdb/db/utils/TimeRangeIteratorTest.java | 220 +++ .../db/utils/datastructure/TimeSelectorTest.java | 22 + .../java/org/apache/iotdb/session/Session.java | 38 +- .../session/IoTDBSessionDisableMemControlIT.java | 9 +- .../session/IoTDBSessionVectorABDeviceIT.java | 2 +- .../IoTDBSessionVectorAggregationWithUnSeqIT.java | 2 +- .../iotdb/session/IoTDBSessionVectorInsertIT.java | 9 +- thrift-confignode/pom.xml | 2 +- thrift/src/main/thrift/rpc.thrift | 2 + .../write/chunk/AlignedChunkGroupWriterImpl.java | 9 +- .../chunk/NonAlignedChunkGroupWriterImpl.java | 6 + .../iotdb/tsfile/write/chunk/ValueChunkWriter.java | 12 +- .../apache/iotdb/tsfile/write/record/Tablet.java | 15 +- .../iotdb/tsfile/write/TsFileWriteApiTest.java | 99 ++ 325 files changed, 11291 insertions(+), 7888 deletions(-)
