This is an automated email from the ASF dual-hosted git repository.
haonan pushed a change to branch speed_up_recover
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 3bb5b076a71 Merge branch 'master' of github.com:apache/iotdb into
speed_up_recover
add 4c0d4f5e771 Compare in double and fix error message
add 3e11b7bd7ab Fix status code of category or data type mismatch
add efd3420674e Load: Add metrics for active load (#13224)
add c0ad8400ce9 Opt type mismatch error msg & fix Blob and date compare bug
add 60d5a7e60af PipeConsensus: build separate dir for each receiver
thread. (#13234)
add 1f08730d384 Pipe: Support async tsfile load when data syncing between
clusters (#13176)
add d883b100094 Fix the error that columns in sort is not appear in
projection
add 7ab36b57436 Inner space compaction splits large target file (#13097)
add a1e6fab0cdf [region migration] Enable SchemaRegion write retry when
migrating region concurrently (#13261)
add e5b87219c08 [region migration] Ratis delete local peer when
resetPeerList is called and myself not in correct peer list #13282
add 124a25e0efe Support update attribute on standalone version & Enable
cache update / mlog writing of createOrUpdate device & Improved the performance
/ semantic of schema device query & Introduce limit/offset of show device
add 459a25771d0 Merge branch 'master' into speed_up_recover
No new revisions were added by this update.
Summary of changes:
.../it/env/cluster/config/MppCommonConfig.java | 6 +-
.../env/cluster/config/MppSharedCommonConfig.java | 7 +-
.../it/env/remote/config/RemoteCommonConfig.java | 3 +-
.../org/apache/iotdb/itbase/env/CommonConfig.java | 2 +-
.../apache/iotdb/db/it/IoTDBInsertMultiRowIT.java | 2 +-
.../org/apache/iotdb/db/it/utils/TestUtils.java | 2 +-
.../pipe/it/autocreate/IoTDBPipeDataSinkIT.java | 97 +++
.../relational/it/db/it/IoTDBInsertTableIT.java | 63 +-
.../it}/IoTDBMultiIDsWithAttributesTableIT.java | 37 +-
.../it/query/old/IoTDBNestedQueryTableIT.java | 19 +-
.../scalar/IoTDBScalarFunctionTableIT.java | 719 +++++++++++----------
.../IoTDBQueryWithComplexValueFilterTableIT.java | 14 +-
...{IoTDBDeviceQueryIT.java => IoTDBDeviceIT.java} | 58 +-
.../iotdb/session/it/IoTDBSessionRelationalIT.java | 84 +++
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 3 +-
.../client/IoTDBConfigNodeSyncClientManager.java | 6 +-
.../protocol/IoTDBConfigRegionAirGapConnector.java | 2 +
.../protocol/IoTDBConfigRegionConnector.java | 6 +-
.../org/apache/iotdb/consensus/IConsensus.java | 4 +-
.../apache/iotdb/consensus/iot/IoTConsensus.java | 3 +-
.../iotdb/consensus/ratis/RatisConsensus.java | 32 +-
.../iotdb/consensus/simple/SimpleConsensus.java | 3 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 52 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 91 ++-
.../schemaregion/SchemaExecutionVisitor.java | 26 +-
.../iotdb/db/exception/sql/SemanticException.java | 8 +
.../client/IoTDBDataNodeAsyncClientManager.java | 32 +-
.../client/IoTDBDataNodeSyncClientManager.java | 6 +-
.../airgap/IoTDBDataNodeAirGapConnector.java | 2 +
.../async/IoTDBDataRegionAsyncConnector.java | 3 +-
.../thrift/sync/IoTDBDataNodeSyncConnector.java | 6 +-
.../pipeconsensus/PipeConsensusReceiver.java | 61 +-
.../protocol/thrift/IoTDBDataNodeReceiver.java | 32 +-
.../iotdb/db/protocol/session/IClientSession.java | 8 +-
.../iotdb/db/queryengine/common/SessionInfo.java | 27 +-
.../execution/exchange/source/SourceHandle.java | 12 +
.../execution/executor/RegionWriteExecutor.java | 21 +-
.../fragment/FragmentInstanceContext.java | 10 +-
.../operator/process/FilterAndProjectOperator.java | 6 +-
.../operator/schema/SchemaQueryScanOperator.java | 73 +--
.../schema/source/DeviceAttributeUpdater.java | 158 +++++
.../schema/source/DevicePredicateFilter.java | 52 +-
.../schema/source/DevicePredicateHandler.java | 132 ++++
.../schema/source/TableDeviceQuerySource.java | 59 +-
.../execution/operator/source/FileLoaderUtils.java | 18 +
.../execution/operator/source/SeriesScanUtil.java | 16 +-
.../relational/ColumnTransformerBuilder.java | 3 +-
.../execution/schedule/AbstractDriverThread.java | 10 +-
.../db/queryengine/plan/analyze/AnalyzeUtils.java | 9 +-
.../queryengine/plan/analyze/AnalyzeVisitor.java | 4 +-
.../plan/analyze/LoadTsFileAnalyzer.java | 4 +-
.../queryengine/plan/execution/QueryExecution.java | 12 +
.../TableModelStatementMemorySourceVisitor.java | 6 +-
.../multi/builtin/helper/CastFunctionHelper.java | 6 +-
.../plan/planner/TableOperatorGenerator.java | 58 +-
.../plan/planner/plan/node/PlanNodeType.java | 10 +-
.../plan/planner/plan/node/PlanVisitor.java | 16 +-
.../plan/node/load/LoadTsFilePieceNode.java | 2 +-
.../read/AbstractTableDeviceQueryNode.java | 7 +-
.../read/TableDeviceAttributeUpdateNode.java | 355 ++++++++++
.../metadata/read/TableDeviceQueryScanNode.java | 40 +-
.../node/metadata/write/CreateTimeSeriesNode.java | 4 +-
.../plan/relational/analyzer/Analysis.java | 4 +-
.../relational/analyzer/StatementAnalyzer.java | 165 +++--
.../schema/CheckSchemaPredicateVisitor.java | 4 +-
.../schema/ExtractPredicateColumnNameVisitor.java | 28 +-
.../relational/metadata/TableMetadataImpl.java | 49 +-
.../metadata/fetcher/TableDeviceSchemaFetcher.java | 49 +-
.../fetcher/TableDeviceSchemaValidator.java | 133 ++--
.../fetcher/TableHeaderSchemaValidator.java | 17 +-
.../fetcher/cache/TableDeviceCacheEntry.java | 19 +-
.../fetcher/cache/TableDeviceSchemaCache.java | 39 +-
.../plan/relational/planner/LogicalPlanner.java | 53 +-
.../rule/MergeLimitOverProjectWithMergeSort.java | 4 +-
.../rule/MergeLimitOverProjectWithSort.java | 16 +-
.../iterative/rule/MergeLimitWithMergeSort.java | 8 +-
.../planner/iterative/rule/MergeLimitWithSort.java | 13 +-
...ode.java => CreateOrUpdateTableDeviceNode.java} | 38 +-
.../sql/ast/AbstractQueryDeviceWithCache.java | 24 +-
.../relational/sql/ast/AbstractTraverseDevice.java | 83 ++-
.../plan/relational/sql/ast/AstVisitor.java | 2 +-
.../plan/relational/sql/ast/CountDevice.java | 8 +-
...CreateDevice.java => CreateOrUpdateDevice.java} | 6 +-
.../plan/relational/sql/ast/ShowDevice.java | 59 +-
.../plan/relational/sql/ast/Update.java | 62 +-
.../plan/relational/sql/ast/UpdateAssignment.java | 32 +-
.../relational/sql/ast/WrappedInsertStatement.java | 21 +-
.../plan/relational/sql/parser/AstBuilder.java | 86 +--
.../plan/relational/sql/util/SqlFormatter.java | 2 +-
.../plan/scheduler/load/LoadTsFileScheduler.java | 12 +-
.../plan/statement/crud/InsertRowStatement.java | 4 +-
.../binary/CompareBinaryColumnTransformer.java | 3 +-
.../scalar/CastFunctionColumnTransformer.java | 149 ++++-
.../schemaengine/schemaregion/ISchemaRegion.java | 11 +-
.../schemaregion/SchemaRegionPlanType.java | 4 +
.../schemaregion/SchemaRegionPlanVisitor.java | 55 +-
.../attribute/DeviceAttributeStore.java | 13 +-
.../attribute/IDeviceAttributeStore.java | 13 +-
.../schemaregion/impl/SchemaRegionMemoryImpl.java | 558 ++++++++++------
.../schemaregion/impl/SchemaRegionPBTreeImpl.java | 21 +-
.../visitor/SchemaRegionPlanDeserializer.java | 109 ++--
.../visitor/SchemaRegionPlanSerializer.java | 182 +++---
.../mtree/impl/mem/MTreeBelowSGMemoryImpl.java | 24 +-
.../write/req/SchemaRegionWritePlanFactory.java | 6 +
.../db/service/metrics/DataNodeMetricsHelper.java | 6 +-
.../iotdb/db/storageengine/StorageEngine.java | 4 +-
.../db/storageengine/dataregion/DataRegion.java | 2 +-
.../performer/impl/FastCompactionPerformer.java | 5 +-
.../impl/ReadChunkCompactionPerformer.java | 118 ++--
.../impl/ReadPointCompactionPerformer.java | 5 +-
.../execute/task/CompactionTaskSummary.java | 2 +-
.../execute/task/InnerSpaceCompactionTask.java | 567 ++++++++++------
.../task/RepairUnsortedFileCompactionTask.java | 65 +-
.../execute/task/SettleCompactionTask.java | 105 +--
.../utils/CompactionTableSchemaCollector.java | 13 +-
.../compaction/execute/utils/CompactionUtils.java | 19 +
.../fast/FastAlignedSeriesCompactionExecutor.java | 4 +-
.../FastNonAlignedSeriesCompactionExecutor.java | 4 +-
.../executor/fast/SeriesCompactionExecutor.java | 6 +-
.../executor/fast/element/AlignedPageElement.java | 5 +-
.../fast/element/ChunkMetadataElement.java | 10 +-
.../utils/executor/fast/element/FileElement.java | 10 +
.../fast/element/NonAlignedPageElement.java | 5 +-
.../utils/executor/fast/element/PageElement.java | 16 +-
.../utils/executor/fast/element/PointElement.java | 8 +-
.../execute/utils/reader/PointPriorityReader.java | 12 +-
.../utils/writer/AbstractCompactionWriter.java | 18 +-
.../writer/AbstractCrossCompactionWriter.java | 13 +-
.../writer/AbstractInnerCompactionWriter.java | 99 ++-
.../utils/writer/FastInnerCompactionWriter.java | 10 +-
.../writer/ReadPointInnerCompactionWriter.java | 7 +-
.../writer/RepairUnsortedFileCompactionWriter.java | 1 -
.../compaction/io/CompactionTsFileWriter.java | 3 +-
.../schedule/CompactionScheduleTaskManager.java | 1 +
.../DefaultCompactionTaskComparatorImpl.java | 16 +-
.../constant/InnerSequenceCompactionSelector.java | 16 +-
.../InnerUnsequenceCompactionSelector.java | 16 +-
.../impl/NewSizeTieredCompactionSelector.java | 295 +++++++++
.../impl/RewriteCrossSpaceCompactionSelector.java | 4 +-
.../selector/impl/SettleSelectorImpl.java | 2 +-
.../impl/SizeTieredCompactionSelector.java | 12 +-
.../utils/CrossSpaceCompactionCandidate.java | 2 +-
.../selector/utils/TsFileResourceCandidate.java | 11 +-
.../compaction/settle/SettleRequestHandler.java | 2 +-
.../read/reader/chunk/DiskAlignedChunkLoader.java | 5 +
.../read/reader/chunk/DiskChunkLoader.java | 5 +
.../dataregion/read/reader/common/Element.java | 9 +-
.../read/reader/common/MergeReaderPriority.java | 73 +++
.../read/reader/common/PriorityMergeReader.java | 49 +-
.../storageengine/dataregion/tsfile/TsFileID.java | 34 +-
.../dataregion/tsfile/TsFileResource.java | 6 +-
.../tsfile/generator/TsFileNameGenerator.java | 34 +
.../dataregion/utils/TsFileResourceUtils.java | 3 +
.../load/LoadTsFileManager.java | 17 +-
.../storageengine/load/active/ActiveLoadAgent.java | 51 ++
.../load/active/ActiveLoadDirScanner.java | 77 +--
.../load/active/ActiveLoadMetricsCollector.java | 96 +++
.../load/active/ActiveLoadPendingQueue.java | 11 +-
.../active/ActiveLoadScheduledExecutorService.java | 107 +++
.../load/active/ActiveLoadTsFileLoader.java | 79 +--
.../load/limiter/LoadTsFileRateLimiter.java | 4 +-
.../memory}/LoadTsFileAbstractMemoryBlock.java | 2 +-
.../LoadTsFileAnalyzeSchemaMemoryBlock.java | 4 +-
.../memory}/LoadTsFileDataCacheMemoryBlock.java | 2 +-
.../load/memory}/LoadTsFileMemoryManager.java | 2 +-
.../load/metrics/ActiveLoadingFilesMetricsSet.java | 113 ++++
.../load/metrics}/LoadTsFileCostMetricsSet.java | 2 +-
.../load/metrics}/LoadTsFileMemMetricSet.java | 4 +-
.../load/splitter/AlignedChunkData.java | 2 +-
.../splitter/BatchedAlignedValueChunkData.java | 2 +-
.../load/splitter/ChunkData.java | 2 +-
.../load/splitter/DeletionData.java | 2 +-
.../load/splitter/NonAlignedChunkData.java | 2 +-
.../load/splitter/TsFileData.java | 2 +-
.../load/splitter/TsFileSplitter.java | 2 +-
.../apache/iotdb/db/tools/schema/MLogParser.java | 1 +
.../org/apache/iotdb/db/utils/DateTimeUtils.java | 38 +-
.../SchemaRegionSimpleRecoverTest.java | 81 +++
.../schemaRegion/SchemaRegionTestUtil.java | 30 +-
.../plan/relational/analyzer/SortTest.java | 141 ++--
.../fetcher/cache/TableDeviceSchemaCacheTest.java | 40 +-
.../storageengine/dataregion/DataRegionTest.java | 22 +-
.../BatchedCompactionWithTsFileSplitterTest.java | 6 +-
.../compaction/CompactionSchedulerTest.java | 96 +--
.../CompactionSchedulerWithFastPerformerTest.java | 96 +--
...eCompactionWithFastPerformerValidationTest.java | 18 +-
...actionWithReadPointPerformerValidationTest.java | 16 +-
.../inner/InnerCompactionSchedulerTest.java | 10 +-
.../inner/InnerSequenceCompactionSpeedTest.java | 4 +-
.../inner/InnerSpaceCompactionSelectorTest.java | 2 +-
.../NewSizeTieredCompactionSelectorTest.java | 644 ++++++++++++++++++
.../repair/RepairUnsortedFileCompactionTest.java | 306 ---------
.../repair/RepairUnsortedFileSchedulerTest.java | 384 +++++++++++
.../settle/SettleCompactionSelectorTest.java | 32 +-
.../settle/SettleCompactionTaskTest.java | 4 +-
.../settle/SettleRequestHandlerTest.java | 6 +-
.../compaction/utils/CompactionConfigRestorer.java | 4 +-
.../compaction/utils/MergeReaderPriorityTest.java | 96 +++
.../conf/iotdb-system.properties.template | 33 +-
.../iotdb/commons/concurrent/ThreadName.java | 2 +
.../iotdb/commons/partition/SchemaPartition.java | 1 +
.../config/constant/PipeConnectorConstant.java | 11 +
.../connector/client/IoTDBSyncClientManager.java | 8 +-
.../common/PipeTransferHandshakeConstant.java | 1 +
.../pipe/connector/protocol/IoTDBConnector.java | 20 +
.../connector/protocol/IoTDBSslSyncConnector.java | 6 +-
.../commons/pipe/receiver/IoTDBFileReceiver.java | 14 +
.../schema/table/column/TsTableColumnSchema.java | 4 +
.../iotdb/commons/service/metric/enums/Metric.java | 1 +
.../org/apache/iotdb/commons/utils/FileUtils.java | 53 ++
210 files changed, 6730 insertions(+), 2640 deletions(-)
rename
integration-test/src/test/java/org/apache/iotdb/relational/it/{query/old/orderBy
=> db/it}/IoTDBMultiIDsWithAttributesTableIT.java (93%)
rename
integration-test/src/test/java/org/apache/iotdb/relational/it/schema/{IoTDBDeviceQueryIT.java
=> IoTDBDeviceIT.java} (74%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceAttributeUpdater.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandler.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/TableDeviceAttributeUpdateNode.java
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/{CreateTableDeviceNode.java
=> CreateOrUpdateTableDeviceNode.java} (88%)
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/{CreateDevice.java
=> CreateOrUpdateDevice.java} (95%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/NewSizeTieredCompactionSelector.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/common/MergeReaderPriority.java
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{queryengine/execution =>
storageengine}/load/LoadTsFileManager.java (97%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadAgent.java
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{queryengine/execution =>
storageengine}/load/active/ActiveLoadDirScanner.java (74%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadMetricsCollector.java
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{queryengine/execution =>
storageengine}/load/active/ActiveLoadPendingQueue.java (81%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadScheduledExecutorService.java
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{queryengine/execution =>
storageengine}/load/active/ActiveLoadTsFileLoader.java (79%)
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{queryengine/execution =>
storageengine}/load/limiter/LoadTsFileRateLimiter.java (96%)
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{queryengine/load
=> storageengine/load/memory}/LoadTsFileAbstractMemoryBlock.java (97%)
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{queryengine/load
=> storageengine/load/memory}/LoadTsFileAnalyzeSchemaMemoryBlock.java (96%)
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{queryengine/load
=> storageengine/load/memory}/LoadTsFileDataCacheMemoryBlock.java (98%)
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{queryengine/load
=> storageengine/load/memory}/LoadTsFileMemoryManager.java (99%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/metrics/ActiveLoadingFilesMetricsSet.java
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{queryengine/metric/load
=> storageengine/load/metrics}/LoadTsFileCostMetricsSet.java (98%)
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{queryengine/metric/load
=> storageengine/load/metrics}/LoadTsFileMemMetricSet.java (96%)
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{queryengine/execution =>
storageengine}/load/splitter/AlignedChunkData.java (99%)
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{queryengine/execution =>
storageengine}/load/splitter/BatchedAlignedValueChunkData.java (99%)
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{queryengine/execution =>
storageengine}/load/splitter/ChunkData.java (97%)
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{queryengine/execution =>
storageengine}/load/splitter/DeletionData.java (97%)
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{queryengine/execution =>
storageengine}/load/splitter/NonAlignedChunkData.java (99%)
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{queryengine/execution =>
storageengine}/load/splitter/TsFileData.java (95%)
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{queryengine/execution =>
storageengine}/load/splitter/TsFileSplitter.java (99%)
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/sizetiered/NewSizeTieredCompactionSelectorTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairUnsortedFileSchedulerTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/MergeReaderPriorityTest.java
