This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch ty/TableModelGrammar in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 54e566c2af21d3150ea6142006f9a61a62e1d0a3 Merge: 2cfa04d38c7 28cd29c930e Author: JackieTien97 <[email protected]> AuthorDate: Tue Jun 4 12:17:17 2024 +0800 resolve conflicts .github/workflows/cluster-it-1c1d.yml | 2 +- .github/workflows/multi-language-client.yml | 7 +- .github/workflows/pipe-it-2cluster.yml | 6 +- .github/workflows/unit-test.yml | 2 +- docker/src/main/DockerCompose/entrypoint.sh | 2 +- .../main/DockerCompose/replace-conf-from-env.sh | 13 +- docker/src/main/Dockerfile-1c1d | 14 +- .../java/org/apache/iotdb/JDBCCharsetExample.java | 99 ++ .../apache/iotdb/SubscriptionSessionExample.java | 2 +- .../iotdb/it/env/cluster/ClusterConstant.java | 4 + .../org/apache/iotdb/it/env/cluster/EnvUtils.java | 18 + .../iotdb/it/env/cluster/config/MppBaseConfig.java | 2 +- .../it/env/cluster/node/AbstractNodeWrapper.java | 8 +- .../it/env/cluster/node/ConfigNodeWrapper.java | 12 +- .../iotdb/it/env/cluster/node/DataNodeWrapper.java | 12 +- .../db/it/regionscan/IoTDBActiveRegionScanIT.java | 480 ++++++++ .../db/it/regionscan/IoTDBActiveRegionScanIT2.java | 52 + .../IoTDBActiveRegionScanWithModsIT.java | 188 +++ .../IoTDBActiveRegionScanWithModsIT2.java | 52 + .../db/it/schema/IoTDBAutoCreateSchemaIT.java | 16 +- .../it/schema/IoTDBDisableAutoCreateSchemaIT.java | 74 ++ .../it/autocreate/IoTDBPipeAutoConflictIT.java | 33 + .../iotdb/session/it/IoTDBSessionSimpleIT.java | 2 +- .../it/dual/IoTDBSubscriptionConsumerGroupIT.java | 6 +- .../it/dual/IoTDBSubscriptionTopicIT.java | 3 +- .../it/local/IoTDBSubscriptionBasicIT.java | 11 +- .../it/local/IoTDBSubscriptionIdempotentIT.java | 2 +- .../it/local/IoTDBSubscriptionRestartIT.java | 2 +- .../apache/iotdb/tools/it/ExportSchemaTestIT.java | 149 +++ .../apache/iotdb/tools/it/ImportSchemaTestIT.java | 126 ++ .../main/java/org/apache/iotdb/udf/api/UDTF.java | 5 + .../java/org/apache/iotdb/udf/api/access/Row.java | 2 +- .../java/org/apache/iotdb/udf/api/type/Type.java | 7 +- .../org/apache/iotdb/udf/api/utils/RowImpl.java | 4 +- .../cli/src/assembly/resources/tools/backup.bat | 14 +- .../cli/src/assembly/resources/tools/backup.sh | 7 +- .../src/assembly/resources/tools/collect-info.bat | 6 +- .../src/assembly/resources/tools/collect-info.sh | 6 +- .../tools/{load-tsfile.bat => export-schema.bat} | 67 +- .../src/assembly/resources/tools/export-schema.sh | 57 + .../tools/{load-tsfile.bat => import-schema.bat} | 66 +- .../src/assembly/resources/tools/import-schema.sh | 57 + .../src/assembly/resources/tools/load-tsfile.bat | 67 +- .../src/assembly/resources/tools/load-tsfile.sh | 80 +- .../org/apache/iotdb/tool/AbstractSchemaTool.java | 242 ++++ .../org/apache/iotdb/tool/AbstractTsFileTool.java | 59 +- .../java/org/apache/iotdb/tool/ExportSchema.java | 339 ++++++ .../java/org/apache/iotdb/tool/ExportTsFile.java | 11 +- .../java/org/apache/iotdb/tool/ImportSchema.java | 653 ++++++++++ .../java/org/apache/iotdb/tool/ImportTsFile.java | 554 +++++++++ .../org/apache/iotdb/tool/IoTDBDataBackTool.java | 39 +- .../main/java/org/apache/iotdb/jdbc/Config.java | 3 + .../org/apache/iotdb/jdbc/IoTDBConnection.java | 10 +- .../apache/iotdb/jdbc/IoTDBConnectionParams.java | 12 + .../org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java | 60 +- .../apache/iotdb/jdbc/IoTDBPreparedStatement.java | 16 +- .../java/org/apache/iotdb/jdbc/IoTDBStatement.java | 91 +- .../src/main/java/org/apache/iotdb/jdbc/Utils.java | 16 +- .../org/apache/iotdb/jdbc/IoTDBStatementTest.java | 8 +- .../java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java | 8 + .../main/java/org/apache/iotdb/rpc/RpcUtils.java | 19 + .../java/org/apache/iotdb/rpc/TSStatusCode.java | 8 + .../rpc/subscription/config/ConsumerConstant.java | 32 +- .../subscription/{ => consumer}/AckStrategy.java | 2 +- .../{ => consumer}/AsyncCommitCallback.java | 2 +- .../{ => consumer}/ConsumeListener.java | 2 +- .../subscription/{ => consumer}/ConsumeResult.java | 2 +- .../{ => consumer}/SubscriptionConsumer.java | 376 +++--- .../SubscriptionExecutorServiceManager.java | 281 +++++ .../{ => consumer}/SubscriptionProvider.java | 4 +- .../{ => consumer}/SubscriptionProviders.java | 21 +- .../{ => consumer}/SubscriptionPullConsumer.java | 160 +-- .../{ => consumer}/SubscriptionPushConsumer.java | 249 ++-- iotdb-core/confignode/src/assembly/confignode.xml | 4 +- .../resources/conf/iotdb-confignode.properties | 147 --- .../assembly/resources/sbin/start-confignode.bat | 30 +- .../assembly/resources/sbin/stop-confignode.bat | 10 +- .../src/assembly/resources/sbin/stop-confignode.sh | 8 +- .../iotdb/confignode/conf/ConfigNodeConfig.java | 34 - .../iotdb/confignode/conf/ConfigNodeConstant.java | 1 - .../confignode/conf/ConfigNodeDescriptor.java | 73 +- .../confignode/conf/ConfigNodeStartupCheck.java | 12 + .../consensus/request/ConfigPhysicalPlan.java | 4 - .../request/write/pipe/payload/PipeSetTTLPlan.java | 103 -- .../iotdb/confignode/manager/TTLManager.java | 7 +- .../manager/load/balancer/RouteBalancer.java | 9 +- .../manager/node/ClusterNodeStartUtils.java | 23 +- .../pipe/event/PipeConfigRegionSnapshotEvent.java | 7 +- .../extractor/ConfigRegionListeningFilter.java | 4 +- .../PipeConfigPhysicalPlanPatternParseVisitor.java | 11 +- .../receiver/protocol/IoTDBConfigNodeReceiver.java | 9 - .../persistence/executor/ConfigPlanExecutor.java | 5 +- .../confignode/persistence/pipe/PipeTaskInfo.java | 7 + .../impl/pipe/task/CreatePipeProcedureV2.java | 79 +- .../scheduler/SimpleProcedureScheduler.java | 7 +- .../iotdb/confignode/service/ConfigNode.java | 3 +- .../request/ConfigPhysicalPlanSerDeTest.java | 11 - ...eConfigPhysicalPlanPatternParseVisitorTest.java | 26 +- .../confignode1conf/iotdb-confignode.properties | 31 - ...b-common.properties => iotdb-system.properties} | 13 + .../confignode2conf/iotdb-confignode.properties | 31 - ...b-common.properties => iotdb-system.properties} | 13 + .../confignode3conf/iotdb-confignode.properties | 31 - ...b-common.properties => iotdb-system.properties} | 13 + iotdb-core/consensus/pom.xml | 9 +- .../apache/iotdb/consensus/ConsensusFactory.java | 22 + .../common/request/IConsensusRequest.java | 4 + .../iotdb/consensus/config/ConsensusConfig.java | 19 +- .../consensus/config/PipeConsensusConfig.java | 358 ++++++ .../apache/iotdb/consensus/iot/IoTConsensus.java | 1 + .../consensus/iot/IoTConsensusServerImpl.java | 1 + .../iot/client/AsyncIoTConsensusServiceClient.java | 2 +- .../apache/iotdb/consensus/pipe/PipeConsensus.java | 465 ++++++++ .../consensus/pipe/PipeConsensusPeerManager.java | 154 +++ .../consensus/pipe/PipeConsensusServerImpl.java | 543 +++++++++ .../consensuspipe/ConsensusPipeDispatcher.java} | 31 +- .../pipe/consensuspipe/ConsensusPipeGuardian.java | 9 +- .../pipe/consensuspipe/ConsensusPipeManager.java | 108 ++ .../pipe/consensuspipe/ConsensusPipeName.java | 98 ++ .../pipe/consensuspipe/ConsensusPipeReceiver.java | 12 +- .../pipe/consensuspipe/ConsensusPipeSelector.java | 11 +- .../pipe/consensuspipe/ProgressIndexManager.java} | 22 +- .../pipe/service/PipeConsensusRPCService.java | 97 ++ .../service/PipeConsensusRPCServiceHandler.java} | 37 +- .../pipe/service/PipeConsensusRPCServiceMBean.java | 7 +- .../service/PipeConsensusRPCServiceProcessor.java | 210 ++++ .../ratis/ApplicationStateMachineProxy.java | 3 + .../iotdb/consensus/ratis/RatisConsensus.java | 2 +- iotdb-core/datanode/pom.xml | 5 + .../resources/conf/iotdb-datanode.properties | 316 ----- .../src/assembly/resources/sbin/start-datanode.bat | 48 +- .../src/assembly/resources/sbin/stop-datanode.bat | 10 +- .../src/assembly/resources/sbin/stop-datanode.sh | 7 +- iotdb-core/datanode/src/assembly/server.xml | 4 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 68 +- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 105 +- .../iotdb/db/conf/rest/IoTDBRestServiceConfig.java | 2 - .../db/conf/rest/IoTDBRestServiceDescriptor.java | 9 +- .../db/consensus/DataRegionConsensusImpl.java | 38 + .../IoTConsensusDataRegionStateMachine.java | 3 +- .../metadata/DataTypeMismatchException.java | 13 + .../PipeDataRegionConnectorConstructor.java | 7 + .../PipeDataRegionProcessorConstructor.java | 4 + .../agent/receiver/PipeDataNodeReceiverAgent.java | 13 + .../agent/runtime/PipeDataNodeRuntimeAgent.java | 32 +- .../db/pipe/agent/task/PipeDataNodeTaskAgent.java | 47 + .../pipeconsensus/PipeConsensusAsyncConnector.java | 524 +++++++++ .../pipeconsensus/PipeConsensusSyncConnector.java | 455 +++++++ .../PipeConsensusTabletBatchEventHandler.java | 128 ++ .../PipeConsensusTabletInsertNodeEventHandler.java | 45 + .../PipeConsensusTabletInsertionEventHandler.java | 109 ++ .../PipeConsensusTabletRawEventHandler.java | 45 + .../PipeConsensusTsFileInsertionEventHandler.java | 279 +++++ .../PipeConsensusAsyncBatchReqBuilder.java} | 20 +- .../builder/PipeConsensusSyncBatchReqBuilder.java | 45 + .../PipeConsensusTransferBatchReqBuilder.java | 218 ++++ .../request/PipeConsensusTabletBatchReq.java | 83 ++ .../request/PipeConsensusTabletBinaryReq.java | 126 ++ .../request/PipeConsensusTabletInsertNodeReq.java | 154 +++ .../request/PipeConsensusTsFilePieceReq.java | 79 ++ .../PipeConsensusTsFilePieceWithModReq.java | 80 ++ .../request/PipeConsensusTsFileSealReq.java | 74 ++ .../request/PipeConsensusTsFileSealWithModReq.java | 85 ++ .../consensus/ConsensusPipeDataNodeDispatcher.java | 121 ++ .../ConsensusPipeDataNodeRuntimeAgentGuardian.java | 48 + .../consensus/PipeConsensusSubtaskExecutor.java} | 25 +- .../consensus/ProgressIndexDataNodeManager.java | 128 ++ .../pipe/execution/PipeSubtaskExecutorManager.java | 7 + .../pipeconsensus/PipeConsensusProcessor.java | 103 ++ ...igner.java => SimpleProgressIndexAssigner.java} | 11 +- .../pipeconsensus/PipeConsensusReceiver.java | 1244 ++++++++++++++++++++ .../pipeconsensus/PipeConsensusReceiverAgent.java | 169 +++ .../pipe/task/builder/PipeDataNodeTaskBuilder.java | 2 + .../pipe/task/connection/PipeEventCollector.java | 4 + .../db/protocol/rest/utils/InsertRowDataUtils.java | 20 +- .../v2/handler/StatementConstructionHandler.java | 27 +- .../protocol/rest/v2/impl/RestApiServiceImpl.java | 86 +- .../impl/DataNodeInternalRPCServiceImpl.java | 7 +- .../queryengine/common/TimeseriesSchemaInfo.java | 54 +- .../execution/driver/DataDriverContext.java | 30 + .../fragment/FragmentInstanceContext.java | 10 + .../process/ActiveRegionScanMergeOperator.java | 288 +++++ .../operator/process/TransformOperator.java | 127 +- .../AbstractRegionScanDataSourceOperator.java | 141 +++ .../AbstractRegionScanForActiveDataUtil.java | 199 ++++ .../source/ActiveDeviceRegionScanOperator.java | 102 ++ .../source/ActiveTimeSeriesRegionScanOperator.java | 143 +++ .../source/RegionScanForActiveDeviceUtil.java | 173 +++ .../source/RegionScanForActiveTimeSeriesUtil.java | 172 +++ .../queryengine/plan/analyze/AnalyzeVisitor.java | 7 +- .../plan/analyze/ClusterPartitionFetcher.java | 3 +- .../analyze/cache/schema/DataNodeTTLCache.java | 36 +- .../config/executor/ClusterConfigTaskExecutor.java | 3 +- .../multi/builtin/BuiltInScalarFunctionHelper.java | 6 +- .../multi/builtin/helper/CastFunctionHelper.java | 6 +- .../multi/builtin/helper/DiffFunctionHelper.java | 6 +- .../builtin/helper/ReplaceFunctionHelper.java | 6 +- .../multi/builtin/helper/RoundFunctionHelper.java | 6 +- .../builtin/helper/SubStringFunctionHelper.java | 6 +- .../visitor/IntermediateLayerVisitor.java | 146 ++- .../plan/planner/LocalExecutionPlanner.java | 17 + .../plan/planner/OperatorTreeGenerator.java | 138 ++- .../planner/distribution/ExchangeNodeAdder.java | 4 +- .../plan/planner/distribution/SourceRewriter.java | 9 +- .../plan/planner/plan/node/PlanNode.java | 10 + .../plan/planner/plan/node/PlanNodeType.java | 4 +- .../plan/planner/plan/node/PlanVisitor.java | 4 +- ...rgeNode.java => ActiveRegionScanMergeNode.java} | 52 +- .../plan/node/source/DeviceRegionScanNode.java | 8 +- .../planner/plan/node/source/RegionScanNode.java | 2 + .../plan/node/source/TimeseriesRegionScanNode.java | 70 +- .../plan/node/write/InsertMultiTabletsNode.java | 6 + .../planner/plan/node/write/InsertRowsNode.java | 6 + .../plan/node/write/InsertRowsOfOneDeviceNode.java | 6 + .../plan/scheduler/load/LoadTsFileScheduler.java | 14 + .../statement/metadata/ShowDatabaseStatement.java | 4 +- .../transformation/api/LayerPointReader.java | 54 - .../api/{LayerRowReader.java => LayerReader.java} | 20 +- .../transformation/api/LayerRowWindowReader.java | 3 - ...izableRowRecordListBackedMultiColumnWindow.java | 6 +- ...wRecordListBackedMultiColumnWindowIterator.java | 6 +- ...ticSerializableTVListBackedSingleColumnRow.java | 2 +- .../LayerPointReaderBackedSingleColumnRow.java | 99 -- .../dag/adapter/PointCollectorAdaptor.java | 97 ++ .../dag/builder/EvaluationDAGBuilder.java | 12 +- .../dag/input/ConstantInputReader.java | 114 +- .../transformation/dag/input/IUDFInputDataSet.java | 21 +- .../dag/input/QueryDataSetInputLayer.java | 294 +---- .../dag/input/TsBlockInputDataSet.java | 37 +- .../intermediate/ConstantIntermediateLayer.java | 17 +- .../dag/intermediate/IntermediateLayer.java | 7 +- .../MultiInputColumnIntermediateLayer.java | 797 ------------- .../dag/intermediate/MultiInputLayer.java | 666 +++++++++++ ...InputColumnMultiReferenceIntermediateLayer.java | 797 ------------- ...nputColumnSingleReferenceIntermediateLayer.java | 668 ----------- .../SingleInputMultiReferenceLayer.java | 724 ++++++++++++ .../SingleInputSingleReferenceLayer.java | 648 ++++++++++ .../transformation/dag/memory/SafetyLine.java | 4 + .../dag/transformer/Transformer.java | 102 +- .../binary/ArithmeticAdditionTransformer.java | 7 +- .../binary/ArithmeticBinaryTransformer.java | 31 +- .../binary/ArithmeticDivisionTransformer.java | 7 +- .../binary/ArithmeticModuloTransformer.java | 7 +- .../ArithmeticMultiplicationTransformer.java | 7 +- .../binary/ArithmeticSubtractionTransformer.java | 7 +- .../dag/transformer/binary/BinaryTransformer.java | 273 +++-- .../binary/CompareBinaryTransformer.java | 33 +- .../binary/CompareEqualToTransformer.java | 22 +- .../binary/CompareGreaterEqualTransformer.java | 22 +- .../binary/CompareGreaterThanTransformer.java | 21 +- .../binary/CompareLessEqualTransformer.java | 22 +- .../binary/CompareLessThanTransformer.java | 21 +- .../binary/CompareNonEqualTransformer.java | 21 +- .../transformer/binary/LogicAndTransformer.java | 6 +- .../transformer/binary/LogicBinaryTransformer.java | 259 ++-- .../dag/transformer/binary/LogicOrTransformer.java | 6 +- .../multi/MappableUDFQueryRowTransformer.java | 102 +- .../transformer/multi/UDFQueryRowTransformer.java | 44 +- .../multi/UDFQueryRowWindowTransformer.java | 21 +- .../dag/transformer/multi/UDFQueryTransformer.java | 12 +- .../multi/UniversalUDFQueryTransformer.java | 73 +- .../transformer/ternary/BetweenTransformer.java | 41 +- .../ternary/CompareTernaryTransformer.java | 55 +- .../transformer/ternary/TernaryTransformer.java | 342 +++--- .../unary/ArithmeticNegationTransformer.java | 92 +- .../dag/transformer/unary/InTransformer.java | 144 ++- .../dag/transformer/unary/IsNullTransformer.java | 39 +- .../dag/transformer/unary/LogicNotTransformer.java | 32 +- .../dag/transformer/unary/RegularTransformer.java | 35 +- .../transformer/unary/TransparentTransformer.java | 37 +- .../dag/transformer/unary/UnaryTransformer.java | 70 +- .../unary/scalar/CastFunctionTransformer.java | 493 ++++++-- .../unary/scalar/DiffFunctionTransformer.java | 159 ++- .../unary/scalar/ReplaceFunctionTransformer.java | 42 +- .../unary/scalar/RoundFunctionTransformer.java | 94 +- .../unary/scalar/SubStringFunctionTransformer.java | 53 +- .../transformation/dag/udf/UDTFExecutor.java | 153 ++- .../transformation/dag/util/LayerCacheUtils.java | 189 +-- .../transformation/dag/util/TransformUtils.java | 46 +- .../transformation/dag/util/TypeUtils.java | 79 ++ .../transformation/datastructure/TVColumns.java | 74 ++ .../iterator/ListForwardIterator.java | 14 +- .../iterator/RowListForwardIterator.java | 100 ++ .../iterator/TVListForwardIterator.java | 109 ++ .../row/ElasticSerializableRowList.java | 485 ++++++++ .../row/ElasticSerializableRowRecordList.java | 314 ----- .../datastructure/row/SerializableRowList.java | 373 ++++++ .../row/SerializableRowRecordList.java | 254 ---- .../tv/ElasticSerializableBinaryTVList.java | 160 ++- .../tv/ElasticSerializableTVList.java | 364 +++--- .../tv/SerializableBooleanTVList.java | 75 -- .../datastructure/tv/SerializableDoubleTVList.java | 75 -- .../datastructure/tv/SerializableFloatTVList.java | 75 -- .../datastructure/tv/SerializableIntTVList.java | 73 -- .../datastructure/tv/SerializableLongTVList.java | 75 -- .../datastructure/tv/SerializableTVList.java | 383 +++++- .../BinaryUtils.java} | 47 +- .../schemaengine/metric/SchemaEngineMemMetric.java | 4 +- .../schemaengine/schemaregion/tag/TagLogFile.java | 2 +- .../schemaregion/utils/ResourceByPathUtils.java | 3 +- .../java/org/apache/iotdb/db/service/DataNode.java | 28 +- .../db/service/metrics/CompactionMetrics.java | 2 +- .../iotdb/db/storageengine/StorageEngine.java | 21 +- .../impl/ReadChunkCompactionPerformer.java | 2 +- .../execute/utils/MultiTsFileDeviceIterator.java | 45 +- .../compaction/io/CompactionTsFileReader.java | 119 +- .../compaction/repair/RepairDataFileScanUtil.java | 45 +- .../dataregion/memtable/AbstractMemTable.java | 129 +- .../dataregion/memtable/TsFileProcessor.java | 93 +- .../read/QueryDataSourceForRegionScan.java | 38 +- .../dataregion/read/filescan/IChunkHandle.java | 13 +- .../dataregion/read/filescan/IFileScanHandle.java | 15 +- .../filescan/impl/ClosedFileScanHandleImpl.java | 48 +- .../filescan/impl/DiskAlignedChunkHandleImpl.java | 15 +- .../read/filescan/impl/DiskChunkHandleImpl.java | 51 +- .../filescan/impl/MemAlignedChunkHandleImpl.java | 17 +- .../read/filescan/impl/MemChunkHandleImpl.java | 25 +- .../filescan/impl/UnclosedFileScanHandleImpl.java | 76 +- .../read/filescan/model/AbstractChunkOffset.java | 16 +- .../model/AbstractDeviceChunkMetaData.java | 8 +- .../read/filescan/model/AlignedChunkOffset.java | 19 +- .../filescan/model/AlignedDeviceChunkMetaData.java | 48 +- .../read/filescan/model/ChunkOffset.java | 12 +- .../read/filescan/model/DeviceChunkMetaData.java | 20 +- .../tsfile/generator/TsFileNameGenerator.java | 15 +- .../dataregion/utils/SharedTimeDataBuffer.java | 10 +- .../dataregion/utils/TsFileResourceUtils.java | 6 +- .../event/SubscriptionEventBinaryCache.java | 2 +- .../apache/iotdb/db/utils/ModificationUtils.java | 12 + .../apache/iotdb/db/utils/TypeInferenceUtils.java | 13 - .../schema/SchemaQueryScanOperatorTest.java | 1 + .../distribution/RegionScanPlanningTest.java | 6 +- .../logical/RegionScanLogicalPlannerTest.java | 24 +- .../builder/EvaluationDAGBuilderTest.java | 4 +- .../ElasticSerializableRowListTest.java | 415 +++++++ .../ElasticSerializableRowRecordListTest.java | 294 ----- .../ElasticSerializableTVListTest.java | 352 ++---- .../SerializableBinaryTVListTest.java | 87 -- .../SerializableBooleanTVListTest.java | 85 -- .../SerializableDoubleTVListTest.java | 84 -- .../datastructure/SerializableFloatTVListTest.java | 84 -- .../datastructure/SerializableIntTVListTest.java | 84 -- .../datastructure/SerializableLongTVListTest.java | 84 -- ...dListTest.java => SerializableRowListTest.java} | 119 +- .../datastructure/SerializableTVListTest.java | 452 ++++++- .../TsFileValidationCorrectnessTests.java | 28 + .../inner/InnerSequenceCompactionSpeedTest.java | 126 ++ .../settle/SettleCompactionTaskTest.java | 44 +- .../compaction/utils/CompactionConfigRestorer.java | 2 - .../utils/MultiTsFileDeviceIteratorTest.java | 3 +- .../rescon/memory/ResourceManagerTest.java | 2 +- .../iotdb/db/utils/TypeInferenceUtilsTest.java | 2 +- .../datanode1conf/iotdb-common.properties | 22 - ...datanode.properties => iotdb-system.properties} | 7 +- .../datanode2conf/iotdb-common.properties | 22 - ...datanode.properties => iotdb-system.properties} | 5 + .../datanode3conf/iotdb-common.properties | 22 - ...datanode.properties => iotdb-system.properties} | 7 +- .../src/test/resources/iotdb-system.properties | 76 ++ .../metrics/config/MetricConfigDescriptor.java | 73 +- .../iotdb/metrics/config/MetricConfigTest.java | 4 +- iotdb-core/node-commons/pom.xml | 5 + ...b-common.properties => iotdb-system.properties} | 748 +++++++++++- .../assembly/resources/sbin/destroy-confignode.bat | 6 +- .../assembly/resources/sbin/destroy-confignode.sh | 7 +- .../assembly/resources/sbin/destroy-datanode.bat | 40 +- .../assembly/resources/sbin/destroy-datanode.sh | 10 +- .../src/assembly/resources/sbin/health_check.bat | 76 +- .../src/assembly/resources/sbin/health_check.sh | 7 +- .../src/assembly/resources/sbin/iotdb-common.sh | 26 +- .../iotdb/commons/client/ClientPoolFactory.java | 72 ++ .../async/AsyncPipeConsensusServiceClient.java} | 57 +- .../container/PipeConsensusClientMgrContainer.java | 91 ++ .../property/PipeConsensusClientProperty.java | 99 ++ .../sync/SyncPipeConsensusServiceClient.java | 139 +++ .../iotdb/commons/concurrent/ThreadName.java | 14 + .../apache/iotdb/commons/conf/CommonConfig.java | 7 +- .../commons/conf/ConfigFileAutoUpdateTool.java | 126 ++ .../iotdb/commons/consensus/ConsensusGroupId.java | 29 + .../consensus/index/impl/HybridProgressIndex.java | 4 + .../consensus/index/impl/RecoverProgressIndex.java | 4 + .../org/apache/iotdb/commons/path/AlignedPath.java | 5 +- .../apache/iotdb/commons/path/MeasurementPath.java | 3 + .../config/constant/PipeConnectorConstant.java | 2 + .../config/constant/PipeExtractorConstant.java | 7 + .../request/PipeConsensusRequestType.java | 67 ++ .../request/PipeConsensusRequestVersion.java | 17 +- .../request/PipeConsensusTransferFilePieceReq.java | 139 +++ .../request/PipeConsensusTransferFileSealReq.java | 138 +++ .../PipeConsensusTransferFileSealWithModReq.java | 177 +++ .../PipeConsensusTransferFilePieceResp.java | 110 ++ .../iotdb/commons/pipe/event/EnrichedEvent.java | 27 + .../pipe/plugin/builtin/BuiltinPipePlugin.java | 7 + .../consensus/PipeConsensusAsyncConnector.java} | 31 +- .../pipeconsensus/PipeConsensusProcessor.java} | 31 +- .../commons/pipe/progress/PipeEventCommitter.java | 44 +- .../commons/pipe/task/meta/PipeStaticMeta.java | 1 + .../commons/pipe/task/meta/PipeTemporaryMeta.java | 4 + .../iotdb/commons/pipe/task/meta/PipeType.java | 3 + .../apache/iotdb/commons/schema/ttl/TTLCache.java | 1 + .../apache/iotdb/commons/service/ServiceType.java | 1 + .../iotdb/commons/service/metric/enums/Metric.java | 1 + .../meta/consumer/ConsumerGroupMetaKeeper.java | 2 +- .../subscription/meta/consumer/ConsumerMeta.java | 2 +- .../iotdb/commons/udf/utils/MasterRepairUtil.java | 2 +- .../commons/udf/utils/UDFDataTypeTransformer.java | 12 - .../src/main/thrift/pipeconsensus.thrift | 109 ++ .../iotdb/library/series/util/ConsecutiveUtil.java | 2 +- 408 files changed, 23438 insertions(+), 9550 deletions(-) diff --cc iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/Utils.java index 281c3f10c17,dcda9e3071e..bdea45fb2d9 --- a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/Utils.java +++ b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/Utils.java @@@ -115,12 -117,6 +117,10 @@@ public class Utils if (info.containsKey(Config.TRUST_STORE_PWD)) { params.setTrustStorePwd(info.getProperty(Config.TRUST_STORE_PWD)); } - + if (info.containsKey(Config.SQL_DIALECT)) { + params.setSqlDialect(info.getProperty(Config.SQL_DIALECT)); + } + return params; } diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/driver/DataDriverContext.java index b46fdbab1b3,c30c1154910..74c50b8790b --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/driver/DataDriverContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/driver/DataDriverContext.java @@@ -32,7 -37,9 +37,9 @@@ import java.util.Optional public class DataDriverContext extends DriverContext { // it will be set to null, after being merged into Parent FIContext - private List<PartialPath> paths; + private List<IFullPath> paths; + private QueryDataSourceType queryDataSourceType = null; + private Map<IDeviceID, Boolean> deviceIDToAligned; // it will be set to null, after QueryDataSource being inited private List<DataSourceOperator> sourceOperators; @@@ -46,9 -54,23 +54,23 @@@ super(parentContext.getFragmentInstanceContext(), pipelineId); this.paths = new ArrayList<>(); this.sourceOperators = new ArrayList<>(); + this.deviceIDToAligned = null; + } + + public void setQueryDataSourceType(QueryDataSourceType queryDataSourceType) { + this.queryDataSourceType = queryDataSourceType; + } + + public void setDeviceIDToAligned(Map<IDeviceID, Boolean> deviceIDToAligned) { + this.deviceIDToAligned = deviceIDToAligned; + } + + public void clearDeviceIDToAligned() { + // friendly for gc + deviceIDToAligned = null; } - public void addPath(PartialPath path) { + public void addPath(IFullPath path) { this.paths.add(path); } diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java index 4beb6ae0208,72425e4df36..861d14203ad --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java @@@ -358,7 -356,11 +358,11 @@@ public class FragmentInstanceContext ex this.sourcePaths = sourcePaths; } + public void setDevicePathsToAligned(Map<IDeviceID, Boolean> devicePathsToAligned) { + this.devicePathsToAligned = devicePathsToAligned; + } + - public void initQueryDataSource(List<PartialPath> sourcePaths) throws QueryProcessException { + public void initQueryDataSource(List<IFullPath> sourcePaths) throws QueryProcessException { long startTime = System.nanoTime(); if (sourcePaths == null) { return; @@@ -420,8 -425,12 +427,11 @@@ } } - public void initRegionScanQueryDataSource(List<PartialPath> pathList) - throws QueryProcessException { + public void initRegionScanQueryDataSource(List<IFullPath> pathList) throws QueryProcessException { long startTime = System.nanoTime(); + if (pathList == null) { + return; + } dataRegion.readLock(); try { this.sharedQueryDataSource = diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeTTLCache.java index 5305a4f27a1,07b835896d9..5dea6b9209d --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeTTLCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeTTLCache.java @@@ -18,11 -18,11 +18,13 @@@ */ package org.apache.iotdb.db.queryengine.plan.analyze.cache.schema; + import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.schema.ttl.TTLCache; + import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.commons.utils.TestOnly; +import org.apache.tsfile.file.metadata.IDeviceID; + import java.util.Map; import java.util.concurrent.locks.ReentrantReadWriteLock; @@@ -81,21 -80,20 +84,30 @@@ public class DataNodeTTLCache } } + public long getTTL(IDeviceID deviceID) { + lock.readLock().lock(); + try { + // TODO Tien change this way + return ttlCache.getClosestTTL(deviceID.toString().split(PATH_SEPARATER_NO_REGEX)); + } finally { + lock.readLock().unlock(); + } + } + + public long getTTL(String[] path) { + lock.readLock().lock(); + try { + return ttlCache.getClosestTTL(path); + } finally { + lock.readLock().unlock(); + } + } + /** Get all ttl map under path node. */ - public Map<String, Long> getTTLUnderOneNode(String path) { + public Map<String, Long> getTTLUnderOneNode(String path) throws IllegalPathException { lock.readLock().lock(); try { - return ttlCache.getAllTTLUnderOneNode(path.split(PATH_SEPARATER_NO_REGEX)); + return ttlCache.getAllTTLUnderOneNode(PathUtils.splitPathToDetachedNodes(path)); } finally { lock.readLock().unlock(); } diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java index a0e28a938b8,4789834f35e..98bd2a962b8 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java @@@ -32,11 -31,11 +32,13 @@@ import org.apache.iotdb.db.queryengine. import org.apache.iotdb.db.queryengine.plan.analyze.TypeProvider; import org.apache.iotdb.db.queryengine.plan.planner.memory.PipelineMemoryEstimator; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.TableMetadataImpl; import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegion; + import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSourceType; import org.apache.iotdb.db.utils.SetThreadName; + import org.apache.tsfile.file.metadata.IDeviceID; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@@ -196,8 -186,19 +202,19 @@@ public class LocalExecutionPlanner return estimatedMemorySize; } + private QueryDataSourceType getQueryDataSourceType(DataDriverContext dataDriverContext) { + return dataDriverContext.getQueryDataSourceType().orElse(QueryDataSourceType.SERIES_SCAN); + } + + private Map<IDeviceID, Boolean> collectDevicePathsToAligned(LocalExecutionPlanContext context) { + DataDriverContext dataDriverContext = (DataDriverContext) context.getDriverContext(); + Map<IDeviceID, Boolean> deviceToAlignedMap = dataDriverContext.getDeviceIDToAligned(); + dataDriverContext.clearDeviceIDToAligned(); + return deviceToAlignedMap; + } + - private List<PartialPath> collectSourcePaths(LocalExecutionPlanContext context) { - List<PartialPath> sourcePaths = new ArrayList<>(); + private List<IFullPath> collectSourcePaths(LocalExecutionPlanContext context) { + List<IFullPath> sourcePaths = new ArrayList<>(); context .getPipelineDriverFactories() .forEach( diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java index d8c572ce000,7269d1cdd09..8efe80638d5 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java @@@ -257,6 -262,8 +265,7 @@@ import org.apache.commons.lang3.StringU import org.apache.commons.lang3.Validate; import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.enums.TSDataType; + import org.apache.tsfile.file.metadata.IDeviceID; -import org.apache.tsfile.file.metadata.PlainDeviceID; import org.apache.tsfile.read.TimeValuePair; import org.apache.tsfile.read.common.block.TsBlockBuilder; import org.apache.tsfile.read.filter.basic.Filter; @@@ -3486,4 -3481,128 +3495,129 @@@ public class OperatorTreeGenerator exte return new ExplainAnalyzeOperator( operatorContext, operator, node.getQueryId(), node.isVerbose(), node.getTimeout()); } + + private List<TSDataType> generateRepeatTsDataType(int size, TSDataType type) { + List<TSDataType> dataTypes = new ArrayList<>(); + for (int i = 0; i < size; i++) { + dataTypes.add(type); + } + return dataTypes; + } + + @Override + public Operator visitRegionMerge( + ActiveRegionScanMergeNode node, LocalExecutionPlanContext context) { + List<Operator> operatorList = + node.getChildren().stream() + .map(child -> child.accept(this, context)) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + OperatorContext operatorContext = + context + .getDriverContext() + .addOperatorContext( + context.getNextOperatorId(), + node.getPlanNodeId(), + ActiveRegionScanMergeOperator.class.getName()); + + List<TSDataType> dataTypes; + if (node.isOutputCount()) { + dataTypes = new ArrayList<>(Collections.singleton(TSDataType.INT64)); + } else { + dataTypes = + generateRepeatTsDataType( + node.getChildren().get(0).getOutputColumnNames().size(), TSDataType.TEXT); + } + return new ActiveRegionScanMergeOperator( + operatorContext, + operatorList, + dataTypes, + node.isOutputCount(), + node.isNeedMerge(), + node.getEstimatedSize()); + } + + @Override + public Operator visitDeviceRegionScan( + DeviceRegionScanNode node, LocalExecutionPlanContext context) { + OperatorContext operatorContext = + context + .getDriverContext() + .addOperatorContext( + context.getNextOperatorId(), + node.getPlanNodeId(), + ActiveDeviceRegionScanOperator.class.getSimpleName()); + Filter filter = context.getGlobalTimeFilter(); + Map<IDeviceID, Boolean> deviceIDToAligned = new HashMap<>(); + for (Map.Entry<PartialPath, Boolean> entry : node.getDevicePathsToAligned().entrySet()) { - deviceIDToAligned.put(new PlainDeviceID(entry.getKey().getFullPath()), entry.getValue()); ++ deviceIDToAligned.put( ++ IDeviceID.Factory.DEFAULT_FACTORY.create(entry.getKey().getFullPath()), entry.getValue()); + } + ActiveDeviceRegionScanOperator regionScanOperator = + new ActiveDeviceRegionScanOperator( + operatorContext, node.getPlanNodeId(), deviceIDToAligned, filter); + + DataDriverContext dataDriverContext = (DataDriverContext) context.getDriverContext(); + dataDriverContext.addSourceOperator(regionScanOperator); + dataDriverContext.setDeviceIDToAligned(deviceIDToAligned); + dataDriverContext.setQueryDataSourceType(QueryDataSourceType.DEVICE_REGION_SCAN); + + return regionScanOperator; + } + + @Override + public Operator visitTimeSeriesRegionScan( + TimeseriesRegionScanNode node, LocalExecutionPlanContext context) { + OperatorContext operatorContext = + context + .getDriverContext() + .addOperatorContext( + context.getNextOperatorId(), + node.getPlanNodeId(), + ActiveTimeSeriesRegionScanOperator.class.getSimpleName()); + Filter filter = context.getGlobalTimeFilter(); + DataDriverContext dataDriverContext = (DataDriverContext) context.getDriverContext(); + + Map<IDeviceID, Map<String, TimeseriesSchemaInfo>> timeseriesToSchemaInfo = new HashMap<>(); + for (Map.Entry<PartialPath, Map<PartialPath, List<TimeseriesSchemaInfo>>> entryMap : + node.getDeviceToTimeseriesSchemaInfo().entrySet()) { + Map<String, TimeseriesSchemaInfo> timeseriesSchemaInfoMap = + getTimeseriesSchemaInfoMap(entryMap, dataDriverContext); + timeseriesToSchemaInfo.put( - new PlainDeviceID(entryMap.getKey().getFullPath()), timeseriesSchemaInfoMap); ++ entryMap.getValue().keySet().iterator().next().getIDeviceID(), timeseriesSchemaInfoMap); + } + ActiveTimeSeriesRegionScanOperator regionScanOperator = + new ActiveTimeSeriesRegionScanOperator( + operatorContext, node.getPlanNodeId(), timeseriesToSchemaInfo, filter); + + dataDriverContext.addSourceOperator(regionScanOperator); + dataDriverContext.setQueryDataSourceType(QueryDataSourceType.TIME_SERIES_REGION_SCAN); + + return regionScanOperator; + } + + private static Map<String, TimeseriesSchemaInfo> getTimeseriesSchemaInfoMap( + Map.Entry<PartialPath, Map<PartialPath, List<TimeseriesSchemaInfo>>> entryMap, + DataDriverContext context) { + Map<String, TimeseriesSchemaInfo> timeseriesSchemaInfoMap = new HashMap<>(); + for (Map.Entry<PartialPath, List<TimeseriesSchemaInfo>> entry : + entryMap.getValue().entrySet()) { + PartialPath path = entry.getKey(); - context.addPath(path); ++ context.addPath(IFullPath.convertToIFullPath(path)); + if (path instanceof MeasurementPath) { + timeseriesSchemaInfoMap.put(path.getMeasurement(), entry.getValue().get(0)); + } else if (path instanceof AlignedPath) { + AlignedPath alignedPath = (AlignedPath) path; + List<String> measurementList = alignedPath.getMeasurementList(); + if (measurementList.size() != entry.getValue().size()) { + throw new IllegalArgumentException( + "The size of measurementList and timeseriesSchemaInfoList should be equal in aligned path."); + } + for (int i = 0; i < measurementList.size(); i++) { + timeseriesSchemaInfoMap.put(measurementList.get(i), entry.getValue().get(i)); + } + } + } + return timeseriesSchemaInfoMap; + } } diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java index af31f08474a,b546516be38..2d910f02c17 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java @@@ -476,33 -461,7 +476,33 @@@ public enum PlanNodeType case 94: return TimeseriesRegionScanNode.deserialize(buffer); case 95: - return RegionMergeNode.deserialize(buffer); + return ActiveRegionScanMergeNode.deserialize(buffer); + case 1000: + return org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode + .deserialize(buffer); + case 1001: + return org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode.deserialize( + buffer); + case 1002: + return org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode.deserialize( + buffer); + case 1003: + return org.apache.iotdb.db.queryengine.plan.relational.planner.node.OutputNode.deserialize( + buffer); + case 1004: + return org.apache.iotdb.db.queryengine.plan.relational.planner.node.LimitNode.deserialize( + buffer); + case 1005: + return org.apache.iotdb.db.queryengine.plan.relational.planner.node.OffsetNode.deserialize( + buffer); + case 1006: + return org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode.deserialize( + buffer); + case 1007: + return org.apache.iotdb.db.queryengine.plan.relational.planner.node.MergeSortNode + .deserialize(buffer); + case 1008: + return TopKNode.deserialize(buffer); default: throw new IllegalArgumentException("Invalid node type: " + nodeType); } diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java index 0e99607701e,aa53aed93cc..e4906ad716f --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java @@@ -38,6 -38,8 +38,7 @@@ import org.apache.tsfile.file.metadata. import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; + import org.apache.tsfile.file.metadata.MetadataIndexNode; -import org.apache.tsfile.file.metadata.PlainDeviceID; import org.apache.tsfile.file.metadata.TimeseriesMetadata; import org.apache.tsfile.read.TsFileDeviceIterator; import org.apache.tsfile.read.TsFileSequenceReader; diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java index 58c46ca492f,1de636c2109..fed7fa4ffd1 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java @@@ -40,6 -39,8 +40,7 @@@ import org.apache.tsfile.file.metadata. import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; + import org.apache.tsfile.file.metadata.MetadataIndexNode; -import org.apache.tsfile.file.metadata.PlainDeviceID; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.TsFileDeviceIterator; @@@ -135,7 -139,7 +139,7 @@@ public class RepairDataFileScanUtil Decoder.getDecoderByType(chunkHeader.getEncodingType(), chunkHeader.getDataType()); while (decoder.hasNext(uncompressedPageData)) { long currentTime = decoder.readLong(uncompressedPageData); - checkPreviousTimeAndUpdate(deviceStr, currentTime); - checkPreviousTimeAndUpdate(((PlainDeviceID) device).toStringID(), currentTime); ++ checkPreviousTimeAndUpdate(device, currentTime); } } } @@@ -151,17 -156,16 +156,19 @@@ measurementChunkMetadataListMapIterator.next(); for (Map.Entry<String, List<ChunkMetadata>> measurementChunkMetadataListEntry : measurementChunkMetadataListMap.entrySet()) { - String path = - device + TsFileConstant.PATH_SEPARATOR + measurementChunkMetadataListEntry.getKey(); - String measurement = measurementChunkMetadataListEntry.getKey(); List<ChunkMetadata> chunkMetadataList = measurementChunkMetadataListEntry.getValue(); - checkSingleNonAlignedSeries(reader, path, chunkMetadataList); - checkSingleNonAlignedSeries(reader, measurement, chunkMetadataList); ++ checkSingleNonAlignedSeries( ++ reader, device, measurementChunkMetadataListEntry.getKey(), chunkMetadataList); previousTime = Long.MIN_VALUE; } } } private void checkSingleNonAlignedSeries( - TsFileSequenceReader reader, String path, List<ChunkMetadata> chunkMetadataList) - TsFileSequenceReader reader, String measurement, List<ChunkMetadata> chunkMetadataList) ++ TsFileSequenceReader reader, ++ IDeviceID deviceID, ++ String measurementId, ++ List<ChunkMetadata> chunkMetadataList) throws IOException { for (ChunkMetadata chunkMetadata : chunkMetadataList) { if (chunkMetadata == null || chunkMetadata.getStatistics().getCount() == 0) { @@@ -190,7 -194,7 +197,7 @@@ TSDataType.INT64); while (timeDecoder.hasNext(timeBuffer)) { long currentTime = timeDecoder.readLong(timeBuffer); - checkPreviousTimeAndUpdate(path, currentTime); - checkPreviousTimeAndUpdate(measurement, currentTime); ++ checkPreviousTimeAndUpdate(deviceID, measurementId, currentTime); } } } @@@ -214,9 -218,9 +221,17 @@@ return ByteBuffer.wrap(uncompressedData); } -- private void checkPreviousTimeAndUpdate(String path, long time) { ++ private void checkPreviousTimeAndUpdate(IDeviceID deviceID, String measurementId, long time) { if (previousTime >= time) { -- throw new CompactionLastTimeCheckFailedException(path, time, previousTime); ++ throw new CompactionLastTimeCheckFailedException( ++ deviceID.toString() + TsFileConstant.PATH_SEPARATOR + measurementId, time, previousTime); ++ } ++ previousTime = time; ++ } ++ ++ private void checkPreviousTimeAndUpdate(IDeviceID deviceID, long time) { ++ if (previousTime >= time) { ++ throw new CompactionLastTimeCheckFailedException(deviceID.toString(), time, previousTime); } previousTime = time; } diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java index 02898d98519,74aed59ab88..c43078a24e5 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java @@@ -1778,11 -1805,10 +1795,12 @@@ public class TsFileProcessor new HashMap<>(); flushQueryLock.readLock().lock(); try { - for (PartialPath seriesPath : pathList) { + for (IFullPath seriesPath : pathList) { Map<String, List<IChunkMetadata>> measurementToChunkMetaList = new HashMap<>(); Map<String, List<IChunkHandle>> measurementToChunkHandleList = new HashMap<>(); - long timeLowerBound = getQueryTimeLowerBound(seriesPath.getDevicePath().getNodes()); ++ + // TODO Tien change the way + long timeLowerBound = getQueryTimeLowerBound(seriesPath.getDeviceId()); for (IMemTable flushingMemTable : flushingMemTables) { if (flushingMemTable.isSignalMemTable()) { continue; @@@ -1793,29 -1819,33 +1811,33 @@@ measurementToChunkMetaList, measurementToChunkHandleList, modsToMemtable); - if (workMemTable != null) { - workMemTable.queryForSeriesRegionScan( - seriesPath, - timeLowerBound, - measurementToChunkMetaList, - measurementToChunkHandleList, - null); - } - - // Some memTable have been flushed already, so we need to get the chunk metadata from - // writer and build chunk handle for disk scanning - buildChunkHandleForFlushedMemTable( - ResourceByPathUtils.getResourceInstance(seriesPath) - .getVisibleMetadataListFromWriter( - writer, tsFileResource, queryContext, timeLowerBound), + } + if (workMemTable != null) { + workMemTable.queryForSeriesRegionScan( + seriesPath, + timeLowerBound, measurementToChunkMetaList, - measurementToChunkHandleList); - - IDeviceID devicePath = seriesPath.getDeviceId(); - if (!measurementToChunkHandleList.isEmpty() || !measurementToChunkMetaList.isEmpty()) { - deviceToMemChunkHandleMap.put(devicePath, measurementToChunkHandleList); - deviceToChunkMetadataListMap.put(devicePath, measurementToChunkMetaList); - } + measurementToChunkHandleList, + null); + } - IDeviceID deviceID = DeviceIDFactory.getInstance().getDeviceID(seriesPath.getDevice()); ++ IDeviceID deviceID = seriesPath.getDeviceId(); + // Some memTable have been flushed already, so we need to get the chunk metadata from + // writer and build chunk handle for disk scanning + buildChunkHandleForFlushedMemTable( + deviceID, + ResourceByPathUtils.getResourceInstance(seriesPath) + .getVisibleMetadataListFromWriter( + writer, tsFileResource, queryContext, timeLowerBound), + measurementToChunkMetaList, + measurementToChunkHandleList); + + if (!measurementToChunkHandleList.isEmpty() || !measurementToChunkMetaList.isEmpty()) { + deviceToMemChunkHandleMap + .computeIfAbsent(deviceID, k -> new HashMap<>()) + .putAll(measurementToChunkHandleList); + deviceToChunkMetadataListMap + .computeIfAbsent(deviceID, k -> new HashMap<>()) + .putAll(measurementToChunkMetaList); } } } catch (QueryProcessException | MetadataException | IOException e) { @@@ -1864,9 -1894,11 +1886,9 @@@ flushQueryLock.readLock().lock(); try { for (Map.Entry<IDeviceID, Boolean> entry : devicePathToAligned.entrySet()) { - IDeviceID devicePath = entry.getKey(); + IDeviceID deviceID = entry.getKey(); boolean isAligned = entry.getValue(); - long timeLowerBound = getQueryTimeLowerBound(devicePath); - long timeLowerBound = - getQueryTimeLowerBound( - PathUtils.splitPathToDetachedNodes(((PlainDeviceID) deviceID).toStringID())); ++ long timeLowerBound = getQueryTimeLowerBound(deviceID); Map<String, List<IChunkMetadata>> measurementToChunkMetadataList = new HashMap<>(); Map<String, List<IChunkHandle>> measurementToMemChunkHandleList = new HashMap<>(); for (IMemTable flushingMemTable : flushingMemTables) { diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/impl/UnclosedFileScanHandleImpl.java index ee825370498,20ae0958a87..ba03f69a11a --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/impl/UnclosedFileScanHandleImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/impl/UnclosedFileScanHandleImpl.java @@@ -24,13 -24,11 +24,11 @@@ import org.apache.iotdb.db.storageengin import org.apache.iotdb.db.storageengine.dataregion.read.filescan.model.AbstractChunkOffset; import org.apache.iotdb.db.storageengine.dataregion.read.filescan.model.AbstractDeviceChunkMetaData; import org.apache.iotdb.db.storageengine.dataregion.read.filescan.model.AlignedDeviceChunkMetaData; - import org.apache.iotdb.db.storageengine.dataregion.read.filescan.model.ChunkOffset; import org.apache.iotdb.db.storageengine.dataregion.read.filescan.model.DeviceChunkMetaData; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; -import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.DeviceTimeIndex; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ArrayDeviceTimeIndex; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ITimeIndex; import org.apache.iotdb.db.storageengine.dataregion.utils.TsFileDeviceStartEndTimeIterator; - import org.apache.iotdb.db.utils.ModificationUtils; import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; diff --cc iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/ResourceManagerTest.java index 2543e64ea63,efa285f7082..f1341386d76 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/ResourceManagerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/ResourceManagerTest.java @@@ -328,7 -327,7 +328,7 @@@ public class ResourceManagerTest } assertEquals(10, tsFileResourceManager.getPriorityQueueSize()); for (int i = 0; i < seqFileNum; i++) { -- // TODO: size of PlainDeviceID may different with string device ++ // TODO: size of DeviceID may different with string device if (i < 8) { assertEquals( TimeIndexLevel.FILE_TIME_INDEX,
