This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch rc/1.2.2 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 0893e51a318efb52053d198e9e2c2cebe0605a6a Author: HTHou <[email protected]> AuthorDate: Tue Sep 26 19:25:35 2023 +0800 Revert "[To rel/1.2][IOTDB-6162] Enhanced accuracy in deleting data (#11202)" This reverts commit ed18e929b8fc06566fda8c79871c829fd43c1ce6. --- .../manager/partition/PartitionManager.java | 2 +- .../dataregion/DataExecutionVisitor.java | 2 +- .../protocol/thrift/impl/ClientRPCServiceImpl.java | 8 ++- .../execution/load/AlignedChunkData.java | 2 +- .../execution/load/NonAlignedChunkData.java | 2 +- .../queryengine/execution/load/TsFileSplitter.java | 2 +- .../db/queryengine/plan/analyze/Analysis.java | 6 +- .../queryengine/plan/analyze/AnalyzeVisitor.java | 2 +- .../plan/node/load/LoadSingleTsFileNode.java | 2 +- .../planner/plan/node/write/DeleteDataNode.java | 24 ++++---- .../planner/plan/node/write/InsertRowNode.java | 2 +- .../planner/plan/node/write/InsertRowsNode.java | 2 +- .../plan/node/write/InsertRowsOfOneDeviceNode.java | 2 +- .../planner/plan/node/write/InsertTabletNode.java | 2 +- .../plan/statement/crud/InsertRowStatement.java | 2 +- .../crud/InsertRowsOfOneDeviceStatement.java | 2 +- .../plan/statement/crud/InsertTabletStatement.java | 2 +- .../iotdb/db/storageengine/StorageEngine.java | 23 ++++++- .../db/storageengine/dataregion/DataRegion.java | 72 ++++++++++++++++------ .../dataregion/tsfile/TsFileManager.java | 19 ------ .../tsfile/timeindex/DeviceTimeIndex.java | 9 ++- .../dataregion/tsfile/timeindex/FileTimeIndex.java | 6 +- .../iotdb/db/tools/TsFileSplitByPartitionTool.java | 10 +-- .../apache/iotdb/db}/utils/TimePartitionUtils.java | 24 +------- .../plan/node/write/WritePlanNodeSplitTest.java | 2 +- .../iotdb/db/storageengine/StorageEngineTest.java | 19 +++--- .../storageengine/dataregion/DataRegionTest.java | 56 ++++++++--------- ...eCrossSpaceCompactionWithFastPerformerTest.java | 15 +++-- ...sSpaceCompactionWithReadPointPerformerTest.java | 15 +++-- .../InnerSeqCompactionWithFastPerformerTest.java | 6 +- ...nerSeqCompactionWithReadChunkPerformerTest.java | 6 +- .../settle/SettleRequestHandlerTest.java | 2 +- .../iotdb/commons/partition/DataPartition.java | 17 ++--- 33 files changed, 194 insertions(+), 173 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java index f49fa60f00e..949b6875316 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java @@ -36,7 +36,6 @@ import org.apache.iotdb.commons.partition.DataPartitionTable; import org.apache.iotdb.commons.partition.SchemaPartitionTable; import org.apache.iotdb.commons.partition.executor.SeriesPartitionExecutor; import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.confignode.client.DataNodeRequestType; import org.apache.iotdb.confignode.client.async.AsyncDataNodeClientPool; import org.apache.iotdb.confignode.client.async.handlers.AsyncClientHandler; @@ -86,6 +85,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TGetSeriesSlotListReq; import org.apache.iotdb.confignode.rpc.thrift.TGetTimeSlotListReq; import org.apache.iotdb.confignode.rpc.thrift.TTimeSlotList; import org.apache.iotdb.consensus.exception.ConsensusException; +import org.apache.iotdb.db.utils.TimePartitionUtils; import org.apache.iotdb.mpp.rpc.thrift.TCreateDataRegionReq; import org.apache.iotdb.mpp.rpc.thrift.TCreateSchemaRegionReq; import org.apache.iotdb.rpc.RpcUtils; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java index a5faea10f5f..1e647995a57 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java @@ -192,7 +192,7 @@ public class DataExecutionVisitor extends PlanVisitor<TSStatus, DataRegion> { try { for (PartialPath path : node.getPathList()) { dataRegion.deleteByDevice( - path, node.getDeleteStartTime(), node.getDeleteEndTime(), node.getSearchIndex()); + path, node.getDeleteStartTime(), node.getDeleteEndTime(), node.getSearchIndex(), null); } return StatusUtils.OK; } catch (IOException e) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java index 75165ef3274..09911c15eec 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java @@ -38,7 +38,6 @@ import org.apache.iotdb.commons.service.metric.MetricService; import org.apache.iotdb.commons.service.metric.enums.Metric; import org.apache.iotdb.commons.service.metric.enums.Tag; import org.apache.iotdb.commons.utils.PathUtils; -import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.audit.AuditLogger; import org.apache.iotdb.db.auth.AuthorityChecker; import org.apache.iotdb.db.conf.IoTDBConfig; @@ -108,6 +107,7 @@ import org.apache.iotdb.db.storageengine.rescon.quotas.DataNodeThrottleQuotaMana import org.apache.iotdb.db.storageengine.rescon.quotas.OperationQuota; import org.apache.iotdb.db.utils.QueryDataSetUtils; import org.apache.iotdb.db.utils.SetThreadName; +import org.apache.iotdb.db.utils.TimePartitionUtils; import org.apache.iotdb.metrics.utils.MetricLevel; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -783,7 +783,7 @@ public class ClientRPCServiceImpl implements IClientRPCServiceWithHandler { partitionFetcher.getDataPartitionWithUnclosedTimeRange( Collections.singletonMap(db, Collections.singletonList(queryParam))); List<TRegionReplicaSet> regionReplicaSets = - dataPartition.getDataRegionReplicaSet(deviceId, null); + dataPartition.getDataRegionReplicaSet(deviceId, Collections.emptyList()); // no valid DataRegion if (regionReplicaSets.isEmpty() @@ -990,7 +990,9 @@ public class ClientRPCServiceImpl implements IClientRPCServiceWithHandler { sgNameToQueryParamsMap.put(database, Collections.singletonList(queryParam)); DataPartition dataPartition = partitionFetcher.getDataPartition(sgNameToQueryParamsMap); List<DataRegion> dataRegionList = new ArrayList<>(); - List<TRegionReplicaSet> replicaSets = dataPartition.getDataRegionReplicaSet(deviceId, null); + List<TRegionReplicaSet> replicaSets = + dataPartition.getDataRegionReplicaSet( + deviceId, Collections.singletonList(timePartitionSlot)); for (TRegionReplicaSet region : replicaSets) { dataRegionList.add( StorageEngine.getInstance() diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/AlignedChunkData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/AlignedChunkData.java index 4d64bd0f90c..ed85602c81c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/AlignedChunkData.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/AlignedChunkData.java @@ -20,7 +20,7 @@ package org.apache.iotdb.db.queryengine.execution.load; import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; -import org.apache.iotdb.commons.utils.TimePartitionUtils; +import org.apache.iotdb.db.utils.TimePartitionUtils; import org.apache.iotdb.tsfile.common.conf.TSFileConfig; import org.apache.iotdb.tsfile.exception.write.PageException; import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/NonAlignedChunkData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/NonAlignedChunkData.java index 2b5d20be9e0..59b0a6bf7b8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/NonAlignedChunkData.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/NonAlignedChunkData.java @@ -20,7 +20,7 @@ package org.apache.iotdb.db.queryengine.execution.load; import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; -import org.apache.iotdb.commons.utils.TimePartitionUtils; +import org.apache.iotdb.db.utils.TimePartitionUtils; import org.apache.iotdb.tsfile.common.conf.TSFileConfig; import org.apache.iotdb.tsfile.exception.write.PageException; import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/TsFileSplitter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/TsFileSplitter.java index 55c5bc0aa3a..83bbd744cea 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/TsFileSplitter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/TsFileSplitter.java @@ -20,10 +20,10 @@ package org.apache.iotdb.db.queryengine.execution.load; import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; -import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.storageengine.dataregion.modification.Deletion; import org.apache.iotdb.db.storageengine.dataregion.modification.Modification; import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; +import org.apache.iotdb.db.utils.TimePartitionUtils; import org.apache.iotdb.tsfile.common.conf.TSFileConfig; import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; import org.apache.iotdb.tsfile.common.constant.TsFileConstant; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java index caeb12e3295..c49ee5a0ee6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java @@ -268,11 +268,13 @@ public class Analysis { } public List<TRegionReplicaSet> getPartitionInfo(PartialPath seriesPath, Filter timefilter) { - return dataPartition.getDataRegionReplicaSet(seriesPath.getDevice(), timefilter); + // TODO: (xingtanzjr) implement the calculation of timePartitionIdList + return dataPartition.getDataRegionReplicaSet(seriesPath.getDevice(), null); } public List<TRegionReplicaSet> getPartitionInfo(String deviceName, Filter globalTimeFilter) { - return dataPartition.getDataRegionReplicaSet(deviceName, globalTimeFilter); + // TODO: (xingtanzjr) implement the calculation of timePartitionIdList + return dataPartition.getDataRegionReplicaSet(deviceName, null); } public Statement getStatement() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java index d1e9bfba691..b0d7b144299 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java @@ -36,7 +36,6 @@ import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.commons.schema.view.LogicalViewSchema; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; import org.apache.iotdb.commons.service.metric.PerformanceOverviewMetrics; -import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.confignode.rpc.thrift.TGetDataNodeLocationsResp; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; @@ -137,6 +136,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowVersionStatement; import org.apache.iotdb.db.schemaengine.SchemaConstant; import org.apache.iotdb.db.schemaengine.template.Template; +import org.apache.iotdb.db.utils.TimePartitionUtils; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java index 687a652c363..b28db7d3abf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java @@ -23,7 +23,6 @@ import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; -import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.pipe.agent.PipeAgent; import org.apache.iotdb.db.queryengine.plan.analyze.Analysis; @@ -32,6 +31,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.utils.TimePartitionUtils; import org.apache.iotdb.tsfile.exception.NotImplementedException; import org.apache.iotdb.tsfile.utils.Pair; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/DeleteDataNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/DeleteDataNode.java index 49582ecf942..9a60ca04ca5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/DeleteDataNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/DeleteDataNode.java @@ -35,7 +35,6 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryValue; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALWriteUtils; -import org.apache.iotdb.tsfile.read.filter.TimeFilter; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; import java.io.DataInputStream; @@ -43,6 +42,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -292,17 +292,17 @@ public class DeleteDataNode extends WritePlanNode implements WALEntryValue { Map<TRegionReplicaSet, List<PartialPath>> regionToPatternMap) { for (DeviceSchemaInfo deviceSchemaInfo : schemaTree.getMatchedDevices(devicePattern)) { PartialPath devicePath = deviceSchemaInfo.getDevicePath(); - // regionId is null when data region of devicePath not existed - dataPartition - .getDataRegionReplicaSet( - devicePath.getFullPath(), TimeFilter.between(deleteStartTime, deleteEndTime)) - .stream() - .filter(regionReplicaSet -> regionReplicaSet.getRegionId() != null) - .forEach( - regionReplicaSet -> - regionToPatternMap - .computeIfAbsent(regionReplicaSet, o -> new ArrayList<>()) - .addAll(pathPattern.alterPrefixPath(devicePath))); + // todo implement time slot + for (TRegionReplicaSet regionReplicaSet : + dataPartition.getDataRegionReplicaSet( + devicePath.getFullPath(), Collections.emptyList())) { + // regionId is null when data region of devicePath not existed + if (regionReplicaSet.getRegionId() != null) { + regionToPatternMap + .computeIfAbsent(regionReplicaSet, o -> new ArrayList<>()) + .addAll(pathPattern.alterPrefixPath(devicePath)); + } + } } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java index 10fb5ff8c09..1d865400293 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java @@ -24,7 +24,6 @@ import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.exception.runtime.UnSupportedDataTypeException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.utils.TestOnly; -import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.queryengine.plan.analyze.Analysis; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; @@ -34,6 +33,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryValue; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALWriteUtils; +import org.apache.iotdb.db.utils.TimePartitionUtils; import org.apache.iotdb.db.utils.TypeInferenceUtils; import org.apache.iotdb.tsfile.exception.NotImplementedException; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsNode.java index 6a506f465a6..439fb5a3b48 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsNode.java @@ -23,13 +23,13 @@ import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.consensus.index.ProgressIndex; import org.apache.iotdb.commons.utils.StatusUtils; -import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.queryengine.plan.analyze.Analysis; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; +import org.apache.iotdb.db.utils.TimePartitionUtils; import org.apache.iotdb.tsfile.exception.NotImplementedException; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsOfOneDeviceNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsOfOneDeviceNode.java index 33f90324792..0167bd7de19 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsOfOneDeviceNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsOfOneDeviceNode.java @@ -24,13 +24,13 @@ import org.apache.iotdb.commons.consensus.index.ProgressIndex; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.utils.StatusUtils; -import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.queryengine.plan.analyze.Analysis; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; +import org.apache.iotdb.db.utils.TimePartitionUtils; import org.apache.iotdb.tsfile.exception.NotImplementedException; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java index 7bfae21daf1..a7ecfe5f1ea 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java @@ -23,7 +23,6 @@ import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.utils.TestOnly; -import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.queryengine.plan.analyze.Analysis; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; @@ -34,6 +33,7 @@ import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferVie import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryValue; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALWriteUtils; import org.apache.iotdb.db.utils.QueryDataSetUtils; +import org.apache.iotdb.db.utils.TimePartitionUtils; import org.apache.iotdb.tsfile.exception.NotImplementedException; import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java index 4e16ff3d3be..88e2d77e9ee 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java @@ -23,7 +23,6 @@ import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.schema.view.LogicalViewSchema; -import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.metadata.DataTypeMismatchException; import org.apache.iotdb.db.exception.metadata.PathNotExistException; @@ -34,6 +33,7 @@ import org.apache.iotdb.db.queryengine.plan.analyze.schema.ISchemaValidation; import org.apache.iotdb.db.queryengine.plan.statement.StatementType; import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; import org.apache.iotdb.db.utils.CommonUtils; +import org.apache.iotdb.db.utils.TimePartitionUtils; import org.apache.iotdb.db.utils.TypeInferenceUtils; import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowsOfOneDeviceStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowsOfOneDeviceStatement.java index 018fc961c60..68688cdb4b1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowsOfOneDeviceStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowsOfOneDeviceStatement.java @@ -21,11 +21,11 @@ package org.apache.iotdb.db.queryengine.plan.statement.crud; import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.queryengine.plan.analyze.schema.ISchemaValidation; import org.apache.iotdb.db.queryengine.plan.statement.StatementType; import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; +import org.apache.iotdb.db.utils.TimePartitionUtils; import org.apache.iotdb.tsfile.exception.NotImplementedException; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java index afe969231a8..de226d0e90c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java @@ -22,7 +22,6 @@ package org.apache.iotdb.db.queryengine.plan.statement.crud; import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.schema.view.LogicalViewSchema; -import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.exception.metadata.DataTypeMismatchException; import org.apache.iotdb.db.exception.metadata.PathNotExistException; import org.apache.iotdb.db.exception.sql.SemanticException; @@ -31,6 +30,7 @@ import org.apache.iotdb.db.queryengine.plan.analyze.schema.ISchemaValidation; import org.apache.iotdb.db.queryengine.plan.statement.StatementType; import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; import org.apache.iotdb.db.utils.CommonUtils; +import org.apache.iotdb.db.utils.TimePartitionUtils; import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException; import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java index 1493de273d7..1cbdf6e7397 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java @@ -32,7 +32,6 @@ import org.apache.iotdb.commons.file.SystemFileFactory; import org.apache.iotdb.commons.service.IService; import org.apache.iotdb.commons.service.ServiceType; import org.apache.iotdb.commons.utils.TestOnly; -import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; @@ -103,6 +102,9 @@ public class StorageEngine implements IService { private static final long TTL_CHECK_INTERVAL = 60 * 1000L; private static final WritingMetrics WRITING_METRICS = WritingMetrics.getInstance(); + /** Time range for dividing database, the time unit is the same with IoTDB's TimestampPrecision */ + private static long timePartitionInterval = -1; + /** * a folder (system/databases/ by default) that persist system info. Each database will have a * subfolder under the systemDir. @@ -147,8 +149,23 @@ public class StorageEngine implements IService { } private static void initTimePartition() { - TimePartitionUtils.setTimePartitionInterval( - CommonDescriptor.getInstance().getConfig().getTimePartitionInterval()); + timePartitionInterval = CommonDescriptor.getInstance().getConfig().getTimePartitionInterval(); + } + + public static long getTimePartitionInterval() { + if (timePartitionInterval == -1) { + initTimePartition(); + } + return timePartitionInterval; + } + + public static long getTimePartition(long time) { + if (timePartitionInterval == -1) { + initTimePartition(); + } + return time > 0 || time % timePartitionInterval == 0 + ? time / timePartitionInterval + : time / timePartitionInterval - 1; } /** block insertion if the insertion is rejected by memory control */ diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 6f33c63b42e..0b8ad3e5a3f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -34,7 +34,6 @@ import org.apache.iotdb.commons.service.metric.MetricService; import org.apache.iotdb.commons.service.metric.PerformanceOverviewMetrics; import org.apache.iotdb.commons.utils.CommonDateTimeUtils; import org.apache.iotdb.commons.utils.TestOnly; -import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; @@ -830,7 +829,7 @@ public class DataRegion implements IDataRegionForQuery { return; } // init map - long timePartitionId = TimePartitionUtils.getTimePartitionId(insertRowNode.getTime()); + long timePartitionId = StorageEngine.getTimePartition(insertRowNode.getTime()); if (!lastFlushTimeMap.checkAndCreateFlushedTimePartition(timePartitionId)) { TimePartitionManager.getInstance() @@ -915,7 +914,7 @@ public class DataRegion implements IDataRegionForQuery { int before = loc; // before time partition long beforeTimePartition = - TimePartitionUtils.getTimePartitionId(insertTabletNode.getTimes()[before]); + StorageEngine.getTimePartition(insertTabletNode.getTimes()[before]); // init map if (!lastFlushTimeMap.checkAndCreateFlushedTimePartition(beforeTimePartition)) { @@ -1762,12 +1761,9 @@ public class DataRegion implements IDataRegionForQuery { /** Seperate tsfiles in TsFileManager to sealedList and unsealedList. */ private void separateTsFile( - List<TsFileResource> sealedResource, - List<TsFileResource> unsealedResource, - long startTime, - long endTime) { + List<TsFileResource> sealedResource, List<TsFileResource> unsealedResource) { tsFileManager - .getTsFileList(true, startTime, endTime) + .getTsFileList(true) .forEach( tsFileResource -> { if (tsFileResource.isClosed()) { @@ -1777,7 +1773,7 @@ public class DataRegion implements IDataRegionForQuery { } }); tsFileManager - .getTsFileList(false, startTime, endTime) + .getTsFileList(false) .forEach( tsFileResource -> { if (tsFileResource.isClosed()) { @@ -1793,9 +1789,15 @@ public class DataRegion implements IDataRegionForQuery { * @param startTime * @param endTime * @param searchIndex + * @param timePartitionFilter * @throws IOException */ - public void deleteByDevice(PartialPath pattern, long startTime, long endTime, long searchIndex) + public void deleteByDevice( + PartialPath pattern, + long startTime, + long endTime, + long searchIndex, + TimePartitionFilter timePartitionFilter) throws IOException { if (SettleService.getINSTANCE().getFilesToBeSettledCount().get() != 0) { throw new IOException( @@ -1823,7 +1825,7 @@ public class DataRegion implements IDataRegionForQuery { // write log to impacted working TsFileProcessors List<WALFlushListener> walListeners = - logDeletionInWAL(startTime, endTime, searchIndex, pattern); + logDeletionInWAL(startTime, endTime, searchIndex, pattern, timePartitionFilter); for (WALFlushListener walFlushListener : walListeners) { if (walFlushListener.waitForResult() == WALFlushListener.Status.FAILURE) { @@ -1836,15 +1838,17 @@ public class DataRegion implements IDataRegionForQuery { List<TsFileResource> sealedTsFileResource = new ArrayList<>(); List<TsFileResource> unsealedTsFileResource = new ArrayList<>(); - separateTsFile(sealedTsFileResource, unsealedTsFileResource, startTime, endTime); + separateTsFile(sealedTsFileResource, unsealedTsFileResource); // deviceMatchInfo is used for filter the matched deviceId in TsFileResource // deviceMatchInfo contains the DeviceId means this device matched the pattern Set<String> deviceMatchInfo = new HashSet<>(); - deleteDataInFiles(unsealedTsFileResource, deletion, devicePaths, deviceMatchInfo); + deleteDataInFiles( + unsealedTsFileResource, deletion, devicePaths, timePartitionFilter, deviceMatchInfo); writeUnlock(); hasReleasedLock = true; - deleteDataInFiles(sealedTsFileResource, deletion, devicePaths, deviceMatchInfo); + deleteDataInFiles( + sealedTsFileResource, deletion, devicePaths, timePartitionFilter, deviceMatchInfo); } catch (Exception e) { throw new IOException(e); @@ -1856,7 +1860,13 @@ public class DataRegion implements IDataRegionForQuery { } private List<WALFlushListener> logDeletionInWAL( - long startTime, long endTime, long searchIndex, PartialPath path) { + long startTime, + long endTime, + long searchIndex, + PartialPath path, + TimePartitionFilter timePartitionFilter) { + long timePartitionStartId = StorageEngine.getTimePartition(startTime); + long timePartitionEndId = StorageEngine.getTimePartition(endTime); List<WALFlushListener> walFlushListeners = new ArrayList<>(); if (config.getWalMode() == WALMode.DISABLE) { return walFlushListeners; @@ -1865,13 +1875,19 @@ public class DataRegion implements IDataRegionForQuery { new DeleteDataNode(new PlanNodeId(""), Collections.singletonList(path), startTime, endTime); deleteDataNode.setSearchIndex(searchIndex); for (Map.Entry<Long, TsFileProcessor> entry : workSequenceTsFileProcessors.entrySet()) { - if (TimePartitionUtils.satisfyPartitionId(startTime, endTime, entry.getKey())) { + if (timePartitionStartId <= entry.getKey() + && entry.getKey() <= timePartitionEndId + && (timePartitionFilter == null + || timePartitionFilter.satisfy(databaseName, entry.getKey()))) { WALFlushListener walFlushListener = entry.getValue().logDeleteDataNodeInWAL(deleteDataNode); walFlushListeners.add(walFlushListener); } } for (Map.Entry<Long, TsFileProcessor> entry : workUnsequenceTsFileProcessors.entrySet()) { - if (TimePartitionUtils.satisfyPartitionId(startTime, endTime, entry.getKey())) { + if (timePartitionStartId <= entry.getKey() + && entry.getKey() <= timePartitionEndId + && (timePartitionFilter == null + || timePartitionFilter.satisfy(databaseName, entry.getKey()))) { WALFlushListener walFlushListener = entry.getValue().logDeleteDataNodeInWAL(deleteDataNode); walFlushListeners.add(walFlushListener); } @@ -1884,7 +1900,12 @@ public class DataRegion implements IDataRegionForQuery { Set<PartialPath> devicePaths, long deleteStart, long deleteEnd, + TimePartitionFilter timePartitionFilter, Set<String> deviceMatchInfo) { + if (timePartitionFilter != null + && !timePartitionFilter.satisfy(databaseName, tsFileResource.getTimePartition())) { + return true; + } long fileStartTime = tsFileResource.getTimeIndex().getMinStartTime(); long fileEndTime = tsFileResource.getTimeIndex().getMaxEndTime(); @@ -1945,6 +1966,7 @@ public class DataRegion implements IDataRegionForQuery { Collection<TsFileResource> tsFileResourceList, Deletion deletion, Set<PartialPath> devicePaths, + TimePartitionFilter timePartitionFilter, Set<String> deviceMatchInfo) throws IOException { for (TsFileResource tsFileResource : tsFileResourceList) { @@ -1953,6 +1975,7 @@ public class DataRegion implements IDataRegionForQuery { devicePaths, deletion.getStartTime(), deletion.getEndTime(), + timePartitionFilter, deviceMatchInfo)) { continue; } @@ -2331,7 +2354,7 @@ public class DataRegion implements IDataRegionForQuery { private void updateLastFlushTime(TsFileResource newTsFileResource) { for (String device : newTsFileResource.getDevices()) { long endTime = newTsFileResource.getEndTime(device); - long timePartitionId = TimePartitionUtils.getTimePartitionId(endTime); + long timePartitionId = StorageEngine.getTimePartition(endTime); lastFlushTimeMap.updateFlushedTime(timePartitionId, device, endTime); lastFlushTimeMap.updateGlobalFlushedTime(device, endTime); } @@ -2691,7 +2714,7 @@ public class DataRegion implements IDataRegionForQuery { continue; } // init map - long timePartitionId = TimePartitionUtils.getTimePartitionId(insertRowNode.getTime()); + long timePartitionId = StorageEngine.getTimePartition(insertRowNode.getTime()); if (!lastFlushTimeMap.checkAndCreateFlushedTimePartition(timePartitionId)) { TimePartitionManager.getInstance() @@ -2885,6 +2908,17 @@ public class DataRegion implements IDataRegionForQuery { void call(TsFileProcessor caller, Map<String, Long> updateMap, long systemFlushTime); } + @FunctionalInterface + public interface CompactionRecoverCallBack { + void call(); + } + + @FunctionalInterface + public interface TimePartitionFilter { + + boolean satisfy(String storageGroupName, long timePartitionId); + } + @FunctionalInterface public interface SettleTsFileCallBack { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java index 814c7c9d345..fbc24dce403 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java @@ -19,7 +19,6 @@ package org.apache.iotdb.db.storageengine.dataregion.tsfile; -import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.storageengine.rescon.memory.TsFileResourceManager; import java.io.IOException; @@ -72,24 +71,6 @@ public class TsFileManager { } } - public List<TsFileResource> getTsFileList(boolean sequence, long startTime, long endTime) { - // the iteration of ConcurrentSkipListMap is not concurrent secure - // so we must add read lock here - readLock(); - try { - List<TsFileResource> allResources = new ArrayList<>(); - Map<Long, TsFileResourceList> chosenMap = sequence ? sequenceFiles : unsequenceFiles; - for (Map.Entry<Long, TsFileResourceList> entry : chosenMap.entrySet()) { - if (TimePartitionUtils.satisfyPartitionId(startTime, endTime, entry.getKey())) { - allResources.addAll(entry.getValue().getArrayList()); - } - } - return allResources; - } finally { - readUnlock(); - } - } - public TsFileResourceList getOrCreateSequenceListByTimePartition(long timePartition) { writeLock("getOrCreateSequenceListByTimePartition"); try { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/DeviceTimeIndex.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/DeviceTimeIndex.java index 090f6998f78..48980c4db02 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/DeviceTimeIndex.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/DeviceTimeIndex.java @@ -22,9 +22,9 @@ package org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.utils.SerializeUtils; -import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.exception.PartitionViolationException; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeDevicePathCache; +import org.apache.iotdb.db.storageengine.StorageEngine; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.tsfile.utils.FilePathUtils; import org.apache.iotdb.tsfile.utils.Pair; @@ -239,8 +239,7 @@ public class DeviceTimeIndex implements ITimeIndex { public long getTimePartition(String tsFilePath) { try { if (deviceToIndex != null && !deviceToIndex.isEmpty()) { - return TimePartitionUtils.getTimePartitionId( - startTimes[deviceToIndex.values().iterator().next()]); + return StorageEngine.getTimePartition(startTimes[deviceToIndex.values().iterator().next()]); } String[] filePathSplits = FilePathUtils.splitTsFilePath(tsFilePath); return Long.parseLong(filePathSplits[filePathSplits.length - 2]); @@ -253,7 +252,7 @@ public class DeviceTimeIndex implements ITimeIndex { private long getTimePartitionWithCheck() { long partitionId = SPANS_MULTI_TIME_PARTITIONS_FLAG_ID; for (int index : deviceToIndex.values()) { - long p = TimePartitionUtils.getTimePartitionId(startTimes[index]); + long p = StorageEngine.getTimePartition(startTimes[index]); if (partitionId == SPANS_MULTI_TIME_PARTITIONS_FLAG_ID) { partitionId = p; } else { @@ -262,7 +261,7 @@ public class DeviceTimeIndex implements ITimeIndex { } } - p = TimePartitionUtils.getTimePartitionId(endTimes[index]); + p = StorageEngine.getTimePartition(endTimes[index]); if (partitionId != p) { return SPANS_MULTI_TIME_PARTITIONS_FLAG_ID; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndex.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndex.java index 04515c54f44..53f82435f57 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndex.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndex.java @@ -20,8 +20,8 @@ package org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex; import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.exception.PartitionViolationException; +import org.apache.iotdb.db.storageengine.StorageEngine; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer; import org.apache.iotdb.tsfile.utils.FilePathUtils; @@ -139,8 +139,8 @@ public class FileTimeIndex implements ITimeIndex { } private long getTimePartitionWithCheck() { - long startPartitionId = TimePartitionUtils.getTimePartitionId(startTime); - long endPartitionId = TimePartitionUtils.getTimePartitionId(endTime); + long startPartitionId = StorageEngine.getTimePartition(startTime); + long endPartitionId = StorageEngine.getTimePartition(endTime); if (startPartitionId == endPartitionId) { return startPartitionId; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/TsFileSplitByPartitionTool.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/TsFileSplitByPartitionTool.java index a76b363bbbc..bb0295a2484 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/TsFileSplitByPartitionTool.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/TsFileSplitByPartitionTool.java @@ -21,7 +21,7 @@ package org.apache.iotdb.db.tools; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.commons.utils.TimePartitionUtils; +import org.apache.iotdb.db.storageengine.StorageEngine; import org.apache.iotdb.db.storageengine.dataregion.modification.Deletion; import org.apache.iotdb.db.storageengine.dataregion.modification.Modification; import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; @@ -271,8 +271,8 @@ public class TsFileSplitByPartitionTool implements AutoCloseable { } } } - return TimePartitionUtils.getTimePartitionId(pageHeader.getStartTime()) - != TimePartitionUtils.getTimePartitionId(pageHeader.getEndTime()); + return StorageEngine.getTimePartition(pageHeader.getStartTime()) + != StorageEngine.getTimePartition(pageHeader.getEndTime()); } /** @@ -358,7 +358,7 @@ public class TsFileSplitByPartitionTool implements AutoCloseable { ByteBuffer pageData, Map<Long, ChunkWriterImpl> partitionChunkWriterMap) throws PageException { - long partitionId = TimePartitionUtils.getTimePartitionId(pageHeader.getStartTime()); + long partitionId = StorageEngine.getTimePartition(pageHeader.getStartTime()); getOrDefaultTsFileIOWriter(oldTsFile, partitionId); ChunkWriterImpl chunkWriter = partitionChunkWriterMap.computeIfAbsent(partitionId, v -> new ChunkWriterImpl(schema)); @@ -413,7 +413,7 @@ public class TsFileSplitByPartitionTool implements AutoCloseable { while (batchData.hasCurrent()) { long time = batchData.currentTime(); Object value = batchData.currentValue(); - long partitionId = TimePartitionUtils.getTimePartitionId(time); + long partitionId = StorageEngine.getTimePartition(time); ChunkWriterImpl chunkWriter = partitionChunkWriterMap.computeIfAbsent(partitionId, v -> new ChunkWriterImpl(schema)); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/TimePartitionUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimePartitionUtils.java similarity index 72% rename from iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/TimePartitionUtils.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimePartitionUtils.java index 630b60ce20a..25e27789e60 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/TimePartitionUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimePartitionUtils.java @@ -16,15 +16,13 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.iotdb.commons.utils; +package org.apache.iotdb.db.utils; import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; import org.apache.iotdb.commons.conf.CommonDescriptor; -import org.apache.iotdb.tsfile.read.filter.basic.Filter; +import org.apache.iotdb.commons.utils.TestOnly; public class TimePartitionUtils { - - /** Time range for dividing database, the time unit is the same with IoTDB's TimestampPrecision */ private static long timePartitionInterval = CommonDescriptor.getInstance().getConfig().getTimePartitionInterval(); @@ -56,23 +54,7 @@ public class TimePartitionUtils { return upperBoundOfTimePartition; } - public static long getTimePartitionId(long time) { - return time > 0 || time % timePartitionInterval == 0 - ? time / timePartitionInterval - : time / timePartitionInterval - 1; - } - - public static boolean satisfyPartitionId(long startTime, long endTime, long partitionId) { - return getTimePartitionId(startTime) <= partitionId - && getTimePartitionId(endTime) >= partitionId; - } - - public static boolean satisfyPartitionStartTime(Filter timeFilter, long partitionStartTime) { - return timeFilter == null - || timeFilter.satisfyStartEndTime( - partitionStartTime, partitionStartTime + timePartitionInterval); - } - + @TestOnly public static void setTimePartitionInterval(long timePartitionInterval) { TimePartitionUtils.timePartitionInterval = timePartitionInterval; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/plan/node/write/WritePlanNodeSplitTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/plan/node/write/WritePlanNodeSplitTest.java index 636b6212c97..60247ce24d8 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/plan/node/write/WritePlanNodeSplitTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/plan/node/write/WritePlanNodeSplitTest.java @@ -32,7 +32,6 @@ import org.apache.iotdb.commons.partition.DataPartition; import org.apache.iotdb.commons.partition.DataPartitionQueryParam; import org.apache.iotdb.commons.partition.executor.SeriesPartitionExecutor; import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.queryengine.plan.analyze.Analysis; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; @@ -41,6 +40,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertMultiT import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; +import org.apache.iotdb.db.utils.TimePartitionUtils; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.junit.After; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/StorageEngineTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/StorageEngineTest.java index 9194887df42..024651d9a0b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/StorageEngineTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/StorageEngineTest.java @@ -19,7 +19,6 @@ package org.apache.iotdb.db.storageengine; import org.apache.iotdb.commons.consensus.DataRegionId; -import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.storageengine.dataregion.DataRegion; import com.google.common.collect.Lists; @@ -72,14 +71,14 @@ public class StorageEngineTest { @Test public void testGetTimePartitionId() { - long timePartitionInterval = TimePartitionUtils.getTimePartitionInterval(); - Assert.assertEquals(-2, TimePartitionUtils.getTimePartitionId(-timePartitionInterval - 1)); - Assert.assertEquals(-1, TimePartitionUtils.getTimePartitionId(-timePartitionInterval)); - Assert.assertEquals(-1, TimePartitionUtils.getTimePartitionId(-1)); - Assert.assertEquals(0, TimePartitionUtils.getTimePartitionId(0)); - Assert.assertEquals(0, TimePartitionUtils.getTimePartitionId(1)); - Assert.assertEquals(0, TimePartitionUtils.getTimePartitionId(timePartitionInterval / 2)); - Assert.assertEquals(1, TimePartitionUtils.getTimePartitionId(timePartitionInterval * 2 - 1)); - Assert.assertEquals(2, TimePartitionUtils.getTimePartitionId(timePartitionInterval * 2 + 1)); + long timePartitionInterval = StorageEngine.getTimePartitionInterval(); + Assert.assertEquals(-2, StorageEngine.getTimePartition(-timePartitionInterval - 1)); + Assert.assertEquals(-1, StorageEngine.getTimePartition(-timePartitionInterval)); + Assert.assertEquals(-1, StorageEngine.getTimePartition(-1)); + Assert.assertEquals(0, StorageEngine.getTimePartition(0)); + Assert.assertEquals(0, StorageEngine.getTimePartition(1)); + Assert.assertEquals(0, StorageEngine.getTimePartition(timePartitionInterval / 2)); + Assert.assertEquals(1, StorageEngine.getTimePartition(timePartitionInterval * 2 - 1)); + Assert.assertEquals(2, StorageEngine.getTimePartition(timePartitionInterval * 2 + 1)); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java index 83a5fb700a0..8bd5295941e 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java @@ -175,7 +175,7 @@ public class DataRegionTest { CompressionType.UNCOMPRESSED, Collections.emptyMap())); - dataRegion.deleteByDevice(new PartialPath(deviceId, measurementId), 0, 15L, -1); + dataRegion.deleteByDevice(new PartialPath(deviceId, measurementId), 0, 15L, -1, null); List<TsFileResource> tsfileResourcesForQuery = new ArrayList<>(); for (TsFileProcessor tsfileProcessor : dataRegion.getWorkUnsequenceTsFileProcessors()) { @@ -951,10 +951,10 @@ public class DataRegionTest { } // delete root.vehicle.d2.s0 data in the second file - dataRegion.deleteByDevice(new PartialPath("root.vehicle.d2.s0"), 50, 150, 0); + dataRegion.deleteByDevice(new PartialPath("root.vehicle.d2.s0"), 50, 150, 0, null); // delete root.vehicle.d2.s0 data in the third file - dataRegion.deleteByDevice(new PartialPath("root.vehicle.d2.s0"), 150, 450, 0); + dataRegion.deleteByDevice(new PartialPath("root.vehicle.d2.s0"), 150, 450, 0, null); for (int i = 0; i < dataRegion.getSequenceFileList().size(); i++) { TsFileResource resource = dataRegion.getSequenceFileList().get(i); @@ -1000,10 +1000,10 @@ public class DataRegionTest { tsFileProcessor.getFlushingMemTable().addLast(tsFileProcessor.getWorkMemTable()); // delete data which is in memtable - dataRegion.deleteByDevice(new PartialPath("root.vehicle.d2.s0"), 50, 70, 0); + dataRegion.deleteByDevice(new PartialPath("root.vehicle.d2.s0"), 50, 70, 0, null); // delete data which is not in memtable - dataRegion.deleteByDevice(new PartialPath("root.vehicle.d200.s0"), 50, 70, 0); + dataRegion.deleteByDevice(new PartialPath("root.vehicle.d200.s0"), 50, 70, 0, null); dataRegion.syncCloseAllWorkingTsFileProcessors(); Assert.assertFalse(tsFileResource.getModFile().exists()); @@ -1022,13 +1022,13 @@ public class DataRegionTest { tsFileProcessor.getFlushingMemTable().addLast(tsFileProcessor.getWorkMemTable()); // delete data which is not in flushing memtable - dataRegion.deleteByDevice(new PartialPath("root.vehicle.d0.s0"), 50, 99, 0); - dataRegion.deleteByDevice(new PartialPath("root.vehicle.d200.s0"), 50, 70, 0); + dataRegion.deleteByDevice(new PartialPath("root.vehicle.d0.s0"), 50, 99, 0, null); + dataRegion.deleteByDevice(new PartialPath("root.vehicle.d200.s0"), 50, 70, 0, null); // delete data which is in flushing memtable - dataRegion.deleteByDevice(new PartialPath("root.vehicle.d0.s0"), 50, 100, 0); - dataRegion.deleteByDevice(new PartialPath("root.vehicle.d0.s0"), 50, 150, 0); - dataRegion.deleteByDevice(new PartialPath("root.vehicle.d0.s0"), 100, 190, 0); + dataRegion.deleteByDevice(new PartialPath("root.vehicle.d0.s0"), 50, 100, 0, null); + dataRegion.deleteByDevice(new PartialPath("root.vehicle.d0.s0"), 50, 150, 0, null); + dataRegion.deleteByDevice(new PartialPath("root.vehicle.d0.s0"), 100, 190, 0, null); dataRegion.syncCloseAllWorkingTsFileProcessors(); Assert.assertTrue(tsFileResource.getModFile().exists()); @@ -1046,13 +1046,13 @@ public class DataRegionTest { TsFileResource tsFileResource = dataRegion.getTsFileManager().getTsFileList(true).get(0); // delete data which is not in work memtable - dataRegion.deleteByDevice(new PartialPath("root.vehicle.d0.s0"), 50, 99, 0); - dataRegion.deleteByDevice(new PartialPath("root.vehicle.d200.s0"), 50, 70, 0); + dataRegion.deleteByDevice(new PartialPath("root.vehicle.d0.s0"), 50, 99, 0, null); + dataRegion.deleteByDevice(new PartialPath("root.vehicle.d200.s0"), 50, 70, 0, null); // delete data which is in work memtable - dataRegion.deleteByDevice(new PartialPath("root.vehicle.d0.s0"), 50, 100, 0); - dataRegion.deleteByDevice(new PartialPath("root.vehicle.d0.s0"), 50, 150, 0); - dataRegion.deleteByDevice(new PartialPath("root.vehicle.d0.s0"), 100, 190, 0); + dataRegion.deleteByDevice(new PartialPath("root.vehicle.d0.s0"), 50, 100, 0, null); + dataRegion.deleteByDevice(new PartialPath("root.vehicle.d0.s0"), 50, 150, 0, null); + dataRegion.deleteByDevice(new PartialPath("root.vehicle.d0.s0"), 100, 190, 0, null); dataRegion.syncCloseAllWorkingTsFileProcessors(); Assert.assertFalse(tsFileResource.getModFile().exists()); @@ -1064,11 +1064,11 @@ public class DataRegionTest { dataRegion.insert(buildInsertRowNodeByTSRecord(record)); } // delete data which is not in work memtable - dataRegion.deleteByDevice(new PartialPath("root.vehicle.d0.s0"), 200, 299, 0); - dataRegion.deleteByDevice(new PartialPath("root.vehicle.d200.s0"), 50, 70, 0); + dataRegion.deleteByDevice(new PartialPath("root.vehicle.d0.s0"), 200, 299, 0, null); + dataRegion.deleteByDevice(new PartialPath("root.vehicle.d200.s0"), 50, 70, 0, null); // delete data which is in work memtable - dataRegion.deleteByDevice(new PartialPath("root.vehicle.d0.s0"), 80, 85, 0); + dataRegion.deleteByDevice(new PartialPath("root.vehicle.d0.s0"), 80, 85, 0, null); Assert.assertFalse(tsFileResource.getModFile().exists()); @@ -1077,14 +1077,14 @@ public class DataRegionTest { tsFileProcessor.getFlushingMemTable().addLast(tsFileProcessor.getWorkMemTable()); // delete data which is not in flushing memtable - dataRegion.deleteByDevice(new PartialPath("root.vehicle.d0.s0"), 0, 49, 0); - dataRegion.deleteByDevice(new PartialPath("root.vehicle.d0.s0"), 100, 200, 0); - dataRegion.deleteByDevice(new PartialPath("root.vehicle.d200.s0"), 50, 70, 0); + dataRegion.deleteByDevice(new PartialPath("root.vehicle.d0.s0"), 0, 49, 0, null); + dataRegion.deleteByDevice(new PartialPath("root.vehicle.d0.s0"), 100, 200, 0, null); + dataRegion.deleteByDevice(new PartialPath("root.vehicle.d200.s0"), 50, 70, 0, null); // delete data which is in flushing memtable - dataRegion.deleteByDevice(new PartialPath("root.vehicle.d0.s0"), 25, 50, 0); - dataRegion.deleteByDevice(new PartialPath("root.vehicle.d0.s0"), 50, 80, 0); - dataRegion.deleteByDevice(new PartialPath("root.vehicle.d0.s0"), 99, 150, 0); + dataRegion.deleteByDevice(new PartialPath("root.vehicle.d0.s0"), 25, 50, 0, null); + dataRegion.deleteByDevice(new PartialPath("root.vehicle.d0.s0"), 50, 80, 0, null); + dataRegion.deleteByDevice(new PartialPath("root.vehicle.d0.s0"), 99, 150, 0, null); dataRegion.syncCloseAllWorkingTsFileProcessors(); Assert.assertTrue(tsFileResource.getModFile().exists()); @@ -1107,11 +1107,11 @@ public class DataRegionTest { TsFileResource tsFileResource = dataRegion.getTsFileManager().getTsFileList(true).get(0); // delete data which is not in working memtable - dataRegion.deleteByDevice(new PartialPath("root.vehicle.d0.s0"), 50, 99, 0); - dataRegion.deleteByDevice(new PartialPath("root.vehicle.d200.s0"), 50, 70, 0); + dataRegion.deleteByDevice(new PartialPath("root.vehicle.d0.s0"), 50, 99, 0, null); + dataRegion.deleteByDevice(new PartialPath("root.vehicle.d200.s0"), 50, 70, 0, null); // delete data which is in working memtable - dataRegion.deleteByDevice(new PartialPath("root.vehicle.d199.*"), 50, 500, 0); + dataRegion.deleteByDevice(new PartialPath("root.vehicle.d199.*"), 50, 500, 0, null); dataRegion.syncCloseAllWorkingTsFileProcessors(); Assert.assertFalse(tsFileResource.getModFile().exists()); @@ -1129,7 +1129,7 @@ public class DataRegionTest { TsFileResource tsFileResource = dataRegion.getTsFileManager().getTsFileList(true).get(0); // delete all data which is in flushing memtable - dataRegion.deleteByDevice(new PartialPath("root.vehicle.d0.s0"), 100, 200, 0); + dataRegion.deleteByDevice(new PartialPath("root.vehicle.d0.s0"), 100, 200, 0, null); dataRegion.syncCloseAllWorkingTsFileProcessors(); Assert.assertFalse(tsFileResource.getTsFile().exists()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/RewriteCrossSpaceCompactionWithFastPerformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/RewriteCrossSpaceCompactionWithFastPerformerTest.java index 800a23801bb..8a22594dc8f 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/RewriteCrossSpaceCompactionWithFastPerformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/RewriteCrossSpaceCompactionWithFastPerformerTest.java @@ -587,7 +587,8 @@ public class RewriteCrossSpaceCompactionWithFastPerformerTest extends AbstractCo + "s0"), 0, 1000, - 0); + 0, + null); CrossSpaceCompactionTask task = new CrossSpaceCompactionTask( @@ -612,7 +613,8 @@ public class RewriteCrossSpaceCompactionWithFastPerformerTest extends AbstractCo + "s0"), 0, 1200, - 0); + 0, + null); for (int i = 0; i < seqResources.size(); i++) { TsFileResource resource = seqResources.get(i); resource.resetModFile(); @@ -706,7 +708,8 @@ public class RewriteCrossSpaceCompactionWithFastPerformerTest extends AbstractCo + "s0"), 0, 1000, - 0); + 0, + null); CrossSpaceCompactionTask task = new CrossSpaceCompactionTask( @@ -731,7 +734,8 @@ public class RewriteCrossSpaceCompactionWithFastPerformerTest extends AbstractCo + "s0"), 0, 1200, - 0); + 0, + null); vsgp.deleteByDevice( new PartialPath( COMPACTION_TEST_SG @@ -742,7 +746,8 @@ public class RewriteCrossSpaceCompactionWithFastPerformerTest extends AbstractCo + "s0"), 0, 1800, - 0); + 0, + null); for (int i = 0; i < seqResources.size(); i++) { TsFileResource resource = seqResources.get(i); resource.resetModFile(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/RewriteCrossSpaceCompactionWithReadPointPerformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/RewriteCrossSpaceCompactionWithReadPointPerformerTest.java index ebfe580ef49..d7a7942acfe 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/RewriteCrossSpaceCompactionWithReadPointPerformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/RewriteCrossSpaceCompactionWithReadPointPerformerTest.java @@ -587,7 +587,8 @@ public class RewriteCrossSpaceCompactionWithReadPointPerformerTest extends Abstr + "s0"), 0, 1000, - 0); + 0, + null); CrossSpaceCompactionTask task = new CrossSpaceCompactionTask( @@ -612,7 +613,8 @@ public class RewriteCrossSpaceCompactionWithReadPointPerformerTest extends Abstr + "s0"), 0, 1200, - 0); + 0, + null); for (int i = 0; i < seqResources.size(); i++) { TsFileResource resource = seqResources.get(i); resource.resetModFile(); @@ -706,7 +708,8 @@ public class RewriteCrossSpaceCompactionWithReadPointPerformerTest extends Abstr + "s0"), 0, 1000, - 0); + 0, + null); CrossSpaceCompactionTask task = new CrossSpaceCompactionTask( @@ -731,7 +734,8 @@ public class RewriteCrossSpaceCompactionWithReadPointPerformerTest extends Abstr + "s0"), 0, 1200, - 0); + 0, + null); vsgp.deleteByDevice( new PartialPath( COMPACTION_TEST_SG @@ -742,7 +746,8 @@ public class RewriteCrossSpaceCompactionWithReadPointPerformerTest extends Abstr + "s0"), 0, 1800, - 0); + 0, + null); for (int i = 0; i < seqResources.size(); i++) { TsFileResource resource = seqResources.get(i); resource.resetModFile(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSeqCompactionWithFastPerformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSeqCompactionWithFastPerformerTest.java index dec41bd7848..90876797997 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSeqCompactionWithFastPerformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSeqCompactionWithFastPerformerTest.java @@ -1129,7 +1129,7 @@ public class InnerSeqCompactionWithFastPerformerTest { COMPACTION_TEST_SG); vsgp.getTsFileResourceManager().addAll(sourceResources, true); // delete data before compaction - vsgp.deleteByDevice(new PartialPath(fullPaths[0]), 0, 1000, 0); + vsgp.deleteByDevice(new PartialPath(fullPaths[0]), 0, 1000, 0, null); ICompactionPerformer performer = new FastCompactionPerformer(false); InnerSpaceCompactionTask task = @@ -1144,8 +1144,8 @@ public class InnerSeqCompactionWithFastPerformerTest { task.setSourceFilesToCompactionCandidate(); task.checkValidAndSetMerging(); // delete data during compaction - vsgp.deleteByDevice(new PartialPath(fullPaths[0]), 0, 1200, 0); - vsgp.deleteByDevice(new PartialPath(fullPaths[0]), 0, 1800, 0); + vsgp.deleteByDevice(new PartialPath(fullPaths[0]), 0, 1200, 0, null); + vsgp.deleteByDevice(new PartialPath(fullPaths[0]), 0, 1800, 0, null); for (int i = 0; i < sourceResources.size() - 1; i++) { TsFileResource resource = sourceResources.get(i); resource.resetModFile(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSeqCompactionWithReadChunkPerformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSeqCompactionWithReadChunkPerformerTest.java index d56b0946808..2533ab45600 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSeqCompactionWithReadChunkPerformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSeqCompactionWithReadChunkPerformerTest.java @@ -1092,7 +1092,7 @@ public class InnerSeqCompactionWithReadChunkPerformerTest { COMPACTION_TEST_SG); vsgp.getTsFileResourceManager().addAll(sourceResources, true); // delete data before compaction - vsgp.deleteByDevice(new PartialPath(fullPaths[0]), 0, 1000, 0); + vsgp.deleteByDevice(new PartialPath(fullPaths[0]), 0, 1000, 0, null); InnerSpaceCompactionTask task = new InnerSpaceCompactionTask( @@ -1106,8 +1106,8 @@ public class InnerSeqCompactionWithReadChunkPerformerTest { task.setSourceFilesToCompactionCandidate(); task.checkValidAndSetMerging(); // delete data during compaction - vsgp.deleteByDevice(new PartialPath(fullPaths[0]), 0, 1200, 0); - vsgp.deleteByDevice(new PartialPath(fullPaths[0]), 0, 1800, 0); + vsgp.deleteByDevice(new PartialPath(fullPaths[0]), 0, 1200, 0, null); + vsgp.deleteByDevice(new PartialPath(fullPaths[0]), 0, 1800, 0, null); for (int i = 0; i < sourceResources.size() - 1; i++) { TsFileResource resource = sourceResources.get(i); resource.resetModFile(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/settle/SettleRequestHandlerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/settle/SettleRequestHandlerTest.java index 732424b944a..666cf43a5f4 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/settle/SettleRequestHandlerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/settle/SettleRequestHandlerTest.java @@ -147,7 +147,7 @@ public class SettleRequestHandlerTest { dataRegion.syncCloseAllWorkingTsFileProcessors(); if (i != 2) { dataRegion.deleteByDevice( - new PartialPath(deviceId, measurementId), 3L * i + 1, 3L * i + 1, -1); + new PartialPath(deviceId, measurementId), 3L * i + 1, 3L * i + 1, -1, null); } } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/DataPartition.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/DataPartition.java index c8425398083..45a46db4a49 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/DataPartition.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/DataPartition.java @@ -21,12 +21,10 @@ package org.apache.iotdb.commons.partition; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.common.rpc.thrift.TSeriesPartitionSlot; import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; -import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.utils.PathUtils; -import org.apache.iotdb.commons.utils.TimePartitionUtils; -import org.apache.iotdb.tsfile.read.filter.basic.Filter; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -35,8 +33,6 @@ import java.util.stream.Collectors; // TODO: Remove this class public class DataPartition extends Partition { - private static long timePartitionInterval = - CommonDescriptor.getInstance().getConfig().getTimePartitionInterval(); public static final TRegionReplicaSet NOT_ASSIGNED = new TRegionReplicaSet(); // Map<StorageGroup, Map<TSeriesPartitionSlot, Map<TTimePartitionSlot, List<TRegionMessage>>>> private Map<String, Map<TSeriesPartitionSlot, Map<TTimePartitionSlot, List<TRegionReplicaSet>>>> @@ -71,18 +67,17 @@ public class DataPartition extends Partition { this.dataPartitionMap = dataPartitionMap; } - public List<TRegionReplicaSet> getDataRegionReplicaSet(String deviceName, Filter timeFilter) { + public List<TRegionReplicaSet> getDataRegionReplicaSet( + String deviceName, List<TTimePartitionSlot> timePartitionSlotList) { String storageGroup = getStorageGroupByDevice(deviceName); TSeriesPartitionSlot seriesPartitionSlot = calculateDeviceGroupId(deviceName); if (!dataPartitionMap.containsKey(storageGroup) || !dataPartitionMap.get(storageGroup).containsKey(seriesPartitionSlot)) { return Collections.singletonList(NOT_ASSIGNED); } - return dataPartitionMap.get(storageGroup).get(seriesPartitionSlot).entrySet().stream() - .filter( - entry -> - TimePartitionUtils.satisfyPartitionStartTime(timeFilter, entry.getKey().startTime)) - .flatMap(entry -> entry.getValue().stream()) + // TODO: (xingtanzjr) the timePartitionIdList is ignored + return dataPartitionMap.get(storageGroup).get(seriesPartitionSlot).values().stream() + .flatMap(Collection::stream) .distinct() .collect(Collectors.toList()); }
