This is an automated email from the ASF dual-hosted git repository. yongzao pushed a commit to branch time-precision in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 7d99db6b7a28bc327ab9802aaffe764b3b64760d Author: YongzaoDan <[email protected]> AuthorDate: Mon Nov 6 15:37:25 2023 +0800 bug fix --- .../write/region/CreateRegionGroupsPlan.java | 63 ++++++++++++++++------ .../manager/load/balancer/RegionBalancer.java | 6 ++- .../partition/DatabasePartitionTable.java | 8 ++- .../persistence/partition/PartitionInfo.java | 6 ++- .../statemachine/CreateRegionGroupsProcedure.java | 13 +++-- .../request/ConfigPhysicalPlanSerDeTest.java | 15 +++--- .../confignode/persistence/PartitionInfoTest.java | 13 +++-- .../impl/CreateRegionGroupsProcedureTest.java | 11 ++-- .../org/apache/iotdb/db/audit/AuditLogger.java | 4 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 7 --- .../db/queryengine/plan/parser/ASTVisitor.java | 19 ++++--- .../planner/plan/parameter/SeriesScanOptions.java | 7 +-- .../db/storageengine/dataregion/DataRegion.java | 17 +++--- .../dataregion/flush/MemTableFlushTask.java | 4 +- .../dataregion/tsfile/TsFileResource.java | 4 +- .../org/apache/iotdb/db/utils/DateTimeUtils.java | 14 ----- .../apache/iotdb/commons/conf/CommonConfig.java | 7 +++ .../iotdb/commons/utils/CommonDateTimeUtils.java | 15 ++++++ 18 files changed, 147 insertions(+), 86 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/region/CreateRegionGroupsPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/region/CreateRegionGroupsPlan.java index 14f6d851563..d2d88d39ad9 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/region/CreateRegionGroupsPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/region/CreateRegionGroupsPlan.java @@ -19,6 +19,7 @@ package org.apache.iotdb.confignode.consensus.request.write.region; +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.commons.utils.BasicStructureSerDeUtil; @@ -39,40 +40,48 @@ import java.util.Map.Entry; import java.util.Objects; import java.util.stream.Collectors; -/** Create regions for specific StorageGroups. */ +/** Create regions for specified Databases. */ public class CreateRegionGroupsPlan extends ConfigPhysicalPlan { - // Map<StorageGroupName, List<TRegionReplicaSet>> + // Map<Database, List<TRegionReplicaSet>> protected final Map<String, List<TRegionReplicaSet>> regionGroupMap; + protected final Map<TConsensusGroupId, Long> regionGroupCreateTimeMap; public CreateRegionGroupsPlan() { super(ConfigPhysicalPlanType.CreateRegionGroups); this.regionGroupMap = new HashMap<>(); + this.regionGroupCreateTimeMap = new HashMap<>(); } public CreateRegionGroupsPlan(ConfigPhysicalPlanType type) { super(type); this.regionGroupMap = new HashMap<>(); + this.regionGroupCreateTimeMap = new HashMap<>(); } public Map<String, List<TRegionReplicaSet>> getRegionGroupMap() { return regionGroupMap; } - public void addRegionGroup(String storageGroup, TRegionReplicaSet regionReplicaSet) { + public Map<TConsensusGroupId, Long> getRegionGroupCreateTimeMap() { + return regionGroupCreateTimeMap; + } + + public void addRegionGroup(String database, TRegionReplicaSet regionReplicaSet, long createTime) { regionGroupMap - .computeIfAbsent(storageGroup, regionReplicaSets -> new ArrayList<>()) + .computeIfAbsent(database, regionReplicaSets -> new ArrayList<>()) .add(regionReplicaSet); + regionGroupCreateTimeMap.put(regionReplicaSet.getRegionId(), createTime); } public void planLog(Logger logger) { for (Map.Entry<String, List<TRegionReplicaSet>> regionGroupEntry : regionGroupMap.entrySet()) { - String storageGroup = regionGroupEntry.getKey(); + String database = regionGroupEntry.getKey(); for (TRegionReplicaSet regionReplicaSet : regionGroupEntry.getValue()) { logger.info( - "[CreateRegionGroups] RegionGroup: {}, belonged StorageGroup: {}, on DataNodes: {}", + "[CreateRegionGroups] RegionGroup: {}, belonged database: {}, on DataNodes: {}", regionReplicaSet.getRegionId(), - storageGroup, + database, regionReplicaSet.getDataNodeLocations().stream() .map(TDataNodeLocation::getDataNodeId) .collect(Collectors.toList())); @@ -96,28 +105,46 @@ public class CreateRegionGroupsPlan extends ConfigPhysicalPlan { stream.writeInt(regionGroupMap.size()); for (Entry<String, List<TRegionReplicaSet>> entry : regionGroupMap.entrySet()) { - String storageGroup = entry.getKey(); + String database = entry.getKey(); List<TRegionReplicaSet> regionReplicaSets = entry.getValue(); - BasicStructureSerDeUtil.write(storageGroup, stream); + BasicStructureSerDeUtil.write(database, stream); stream.writeInt(regionReplicaSets.size()); regionReplicaSets.forEach( regionReplicaSet -> ThriftCommonsSerDeUtils.serializeTRegionReplicaSet(regionReplicaSet, stream)); } + + stream.writeInt(regionGroupCreateTimeMap.size()); + for (Entry<TConsensusGroupId, Long> entry : regionGroupCreateTimeMap.entrySet()) { + TConsensusGroupId regionId = entry.getKey(); + long createTime = entry.getValue(); + ThriftCommonsSerDeUtils.serializeTConsensusGroupId(regionId, stream); + stream.writeLong(createTime); + } } @Override protected void deserializeImpl(ByteBuffer buffer) throws IOException { - int storageGroupNum = buffer.getInt(); - for (int i = 0; i < storageGroupNum; i++) { - String storageGroup = BasicStructureSerDeUtil.readString(buffer); - regionGroupMap.put(storageGroup, new ArrayList<>()); + int databaseNum = buffer.getInt(); + for (int i = 0; i < databaseNum; i++) { + String database = BasicStructureSerDeUtil.readString(buffer); + regionGroupMap.put(database, new ArrayList<>()); int regionReplicaSetNum = buffer.getInt(); for (int j = 0; j < regionReplicaSetNum; j++) { TRegionReplicaSet regionReplicaSet = ThriftCommonsSerDeUtils.deserializeTRegionReplicaSet(buffer); - regionGroupMap.get(storageGroup).add(regionReplicaSet); + regionGroupMap.get(database).add(regionReplicaSet); + } + } + + if (buffer.hasRemaining()) { + // For compatibility + int regionGroupCreateTimeMapSize = buffer.getInt(); + for (int i = 0; i < regionGroupCreateTimeMapSize; i++) { + TConsensusGroupId regionId = ThriftCommonsSerDeUtils.deserializeTConsensusGroupId(buffer); + long createTime = buffer.getLong(); + regionGroupCreateTimeMap.put(regionId, createTime); } } } @@ -130,12 +157,16 @@ public class CreateRegionGroupsPlan extends ConfigPhysicalPlan { if (o == null || getClass() != o.getClass()) { return false; } + if (!super.equals(o)) { + return false; + } CreateRegionGroupsPlan that = (CreateRegionGroupsPlan) o; - return regionGroupMap.equals(that.regionGroupMap); + return Objects.equals(regionGroupMap, that.regionGroupMap) + && Objects.equals(regionGroupCreateTimeMap, that.regionGroupCreateTimeMap); } @Override public int hashCode() { - return Objects.hash(regionGroupMap); + return Objects.hash(super.hashCode(), regionGroupMap, regionGroupCreateTimeMap); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/RegionBalancer.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/RegionBalancer.java index c9965fe036c..eb801ba1525 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/RegionBalancer.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/RegionBalancer.java @@ -23,6 +23,7 @@ import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.commons.cluster.NodeStatus; +import org.apache.iotdb.commons.utils.CommonDateTimeUtils; import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; import org.apache.iotdb.confignode.consensus.request.write.region.CreateRegionGroupsPlan; import org.apache.iotdb.confignode.exception.DatabaseNotExistsException; @@ -64,7 +65,7 @@ public class RegionBalancer { /** * Generate a RegionGroups' allocation plan(CreateRegionGroupsPlan) * - * @param allotmentMap Map<StorageGroupName, RegionGroup allotment> + * @param allotmentMap Map<DatabaseName, RegionGroup allotment> * @param consensusGroupType TConsensusGroupType of the new RegionGroups * @return CreateRegionGroupsPlan * @throws NotEnoughDataNodeException When the number of DataNodes is not enough for allocation @@ -119,7 +120,8 @@ public class RegionBalancer { replicationFactor, new TConsensusGroupId( consensusGroupType, getPartitionManager().generateNextRegionGroupId())); - createRegionGroupsPlan.addRegionGroup(database, newRegionGroup); + createRegionGroupsPlan.addRegionGroup( + database, newRegionGroup, CommonDateTimeUtils.currentTime()); // Mark the new RegionGroup as allocated allocatedRegionGroups.add(newRegionGroup); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/DatabasePartitionTable.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/DatabasePartitionTable.java index bdca8aa7959..59364e4d004 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/DatabasePartitionTable.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/DatabasePartitionTable.java @@ -90,12 +90,16 @@ public class DatabasePartitionTable { * Cache allocation result of new RegionGroups. * * @param replicaSets List<TRegionReplicaSet> + * @param regionGroupCreateTimeMap Map<RegionGroupId, create time> */ - public void createRegionGroups(List<TRegionReplicaSet> replicaSets) { + public void createRegionGroups( + List<TRegionReplicaSet> replicaSets, Map<TConsensusGroupId, Long> regionGroupCreateTimeMap) { replicaSets.forEach( replicaSet -> regionGroupMap.put( - replicaSet.getRegionId(), new RegionGroup(System.currentTimeMillis(), replicaSet))); + replicaSet.getRegionId(), + new RegionGroup( + regionGroupCreateTimeMap.get(replicaSet.getRegionId()), replicaSet))); } /** @return Deep copy of all Regions' RegionReplicaSet within one StorageGroup */ diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java index 710c4b91ebf..66034f362f7 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java @@ -164,8 +164,10 @@ public class PartitionInfo implements SnapshotProcessor { plan.getRegionGroupMap() .forEach( - (storageGroup, regionReplicaSets) -> { - databasePartitionTables.get(storageGroup).createRegionGroups(regionReplicaSets); + (database, regionReplicaSets) -> { + databasePartitionTables + .get(database) + .createRegionGroups(regionReplicaSets, plan.getRegionGroupCreateTimeMap()); regionReplicaSets.forEach( regionReplicaSet -> maxRegionId.set( diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/statemachine/CreateRegionGroupsProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/statemachine/CreateRegionGroupsProcedure.java index 9a017a43eba..caefac2a0d0 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/statemachine/CreateRegionGroupsProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/statemachine/CreateRegionGroupsProcedure.java @@ -23,6 +23,7 @@ import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.commons.cluster.RegionStatus; +import org.apache.iotdb.commons.utils.CommonDateTimeUtils; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.commons.utils.ThriftCommonsSerDeUtils; import org.apache.iotdb.confignode.consensus.request.write.region.CreateRegionGroupsPlan; @@ -95,14 +96,15 @@ public class CreateRegionGroupsProcedure createRegionGroupsPlan .getRegionGroupMap() .forEach( - (storageGroup, regionReplicaSets) -> + (database, regionReplicaSets) -> regionReplicaSets.forEach( regionReplicaSet -> { if (!failedRegionReplicaSets.containsKey( regionReplicaSet.getRegionId())) { // A RegionGroup was created successfully when // all RegionReplicas were created successfully - persistPlan.addRegionGroup(storageGroup, regionReplicaSet); + persistPlan.addRegionGroup( + database, regionReplicaSet, CommonDateTimeUtils.currentTime()); LOGGER.info( "[CreateRegionGroups] All replicas of RegionGroup: {} are created successfully!", regionReplicaSet.getRegionId()); @@ -114,7 +116,8 @@ public class CreateRegionGroupsProcedure <= (regionReplicaSet.getDataNodeLocationsSize() - 1) / 2) { // A RegionGroup can provide service as long as there are more than // half of the RegionReplicas created successfully - persistPlan.addRegionGroup(storageGroup, regionReplicaSet); + persistPlan.addRegionGroup( + database, regionReplicaSet, CommonDateTimeUtils.currentTime()); // Build recreate tasks failedRegionReplicas @@ -123,11 +126,11 @@ public class CreateRegionGroupsProcedure targetDataNode -> { RegionCreateTask createTask = new RegionCreateTask( - targetDataNode, storageGroup, regionReplicaSet); + targetDataNode, database, regionReplicaSet); if (TConsensusGroupType.DataRegion.equals( regionReplicaSet.getRegionId().getType())) { try { - createTask.setTTL(env.getTTL(storageGroup)); + createTask.setTTL(env.getTTL(database)); } catch (DatabaseNotExistsException e) { LOGGER.error("Can't get TTL", e); } diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java index 82ff95a64af..02523c99c17 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java @@ -51,6 +51,7 @@ import org.apache.iotdb.commons.sync.PipeStatus; import org.apache.iotdb.commons.sync.TsFilePipeInfo; import org.apache.iotdb.commons.trigger.TriggerInformation; import org.apache.iotdb.commons.udf.UDFInformation; +import org.apache.iotdb.commons.utils.CommonDateTimeUtils; import org.apache.iotdb.confignode.consensus.request.auth.AuthorPlan; import org.apache.iotdb.confignode.consensus.request.read.database.CountDatabasePlan; import org.apache.iotdb.confignode.consensus.request.read.database.GetDatabasePlan; @@ -349,12 +350,12 @@ public class ConfigPhysicalPlanSerDeTest { TRegionReplicaSet dataRegionSet = new TRegionReplicaSet(); dataRegionSet.setRegionId(new TConsensusGroupId(TConsensusGroupType.DataRegion, 0)); dataRegionSet.setDataNodeLocations(Collections.singletonList(dataNodeLocation)); - req0.addRegionGroup("root.sg0", dataRegionSet); + req0.addRegionGroup("root.sg0", dataRegionSet, CommonDateTimeUtils.currentTime()); TRegionReplicaSet schemaRegionSet = new TRegionReplicaSet(); schemaRegionSet.setRegionId(new TConsensusGroupId(TConsensusGroupType.SchemaRegion, 1)); schemaRegionSet.setDataNodeLocations(Collections.singletonList(dataNodeLocation)); - req0.addRegionGroup("root.sg1", schemaRegionSet); + req0.addRegionGroup("root.sg1", schemaRegionSet, CommonDateTimeUtils.currentTime()); CreateRegionGroupsPlan req1 = (CreateRegionGroupsPlan) ConfigPhysicalPlan.Factory.create(req0.serializeToByteBuffer()); @@ -866,11 +867,13 @@ public class ConfigPhysicalPlanSerDeTest { failedRegions.put(dataRegionGroupId, dataRegionSet); failedRegions.put(schemaRegionGroupId, schemaRegionSet); CreateRegionGroupsPlan createRegionGroupsPlan = new CreateRegionGroupsPlan(); - createRegionGroupsPlan.addRegionGroup("root.sg0", dataRegionSet); - createRegionGroupsPlan.addRegionGroup("root.sg1", schemaRegionSet); + createRegionGroupsPlan.addRegionGroup( + "root.sg0", dataRegionSet, CommonDateTimeUtils.currentTime()); + createRegionGroupsPlan.addRegionGroup( + "root.sg1", schemaRegionSet, CommonDateTimeUtils.currentTime()); CreateRegionGroupsPlan persistPlan = new CreateRegionGroupsPlan(); - persistPlan.addRegionGroup("root.sg0", dataRegionSet); - persistPlan.addRegionGroup("root.sg1", schemaRegionSet); + persistPlan.addRegionGroup("root.sg0", dataRegionSet, CommonDateTimeUtils.currentTime()); + persistPlan.addRegionGroup("root.sg1", schemaRegionSet, CommonDateTimeUtils.currentTime()); CreateRegionGroupsProcedure procedure0 = new CreateRegionGroupsProcedure( TConsensusGroupType.DataRegion, createRegionGroupsPlan, persistPlan, failedRegions); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/PartitionInfoTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/PartitionInfoTest.java index f39dbae6a63..2f289dd7246 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/PartitionInfoTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/PartitionInfoTest.java @@ -29,6 +29,7 @@ import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; import org.apache.iotdb.commons.partition.DataPartitionTable; import org.apache.iotdb.commons.partition.SchemaPartitionTable; import org.apache.iotdb.commons.partition.SeriesPartitionTable; +import org.apache.iotdb.commons.utils.CommonDateTimeUtils; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; import org.apache.iotdb.confignode.consensus.request.read.region.GetRegionInfoListPlan; import org.apache.iotdb.confignode.consensus.request.write.database.DatabaseSchemaPlan; @@ -113,7 +114,8 @@ public class PartitionInfoTest { testFlag.SchemaPartition.getFlag(), generateTConsensusGroupId( testFlag.SchemaPartition.getFlag(), TConsensusGroupType.SchemaRegion)); - createRegionGroupsReq.addRegionGroup("root.test", schemaRegionReplicaSet); + createRegionGroupsReq.addRegionGroup( + "root.test", schemaRegionReplicaSet, CommonDateTimeUtils.currentTime()); partitionInfo.createRegionGroups(createRegionGroupsReq); // Create a DataRegion @@ -123,7 +125,8 @@ public class PartitionInfoTest { testFlag.DataPartition.getFlag(), generateTConsensusGroupId( testFlag.DataPartition.getFlag(), TConsensusGroupType.DataRegion)); - createRegionGroupsReq.addRegionGroup("root.test", dataRegionReplicaSet); + createRegionGroupsReq.addRegionGroup( + "root.test", dataRegionReplicaSet, CommonDateTimeUtils.currentTime()); partitionInfo.createRegionGroups(createRegionGroupsReq); // Create a SchemaPartition @@ -168,7 +171,8 @@ public class PartitionInfoTest { testFlag.SchemaPartition.getFlag(), generateTConsensusGroupId( testFlag.SchemaPartition.getFlag(), TConsensusGroupType.SchemaRegion)); - createRegionGroupsPlan.addRegionGroup("root.test" + i, schemaRegionReplicaSet); + createRegionGroupsPlan.addRegionGroup( + "root.test" + i, schemaRegionReplicaSet, CommonDateTimeUtils.currentTime()); partitionInfo.createRegionGroups(createRegionGroupsPlan); // Create a DataRegion @@ -178,7 +182,8 @@ public class PartitionInfoTest { testFlag.DataPartition.getFlag(), generateTConsensusGroupId( testFlag.DataPartition.getFlag(), TConsensusGroupType.DataRegion)); - createRegionGroupsPlan.addRegionGroup("root.test" + i, dataRegionReplicaSet); + createRegionGroupsPlan.addRegionGroup( + "root.test" + i, dataRegionReplicaSet, CommonDateTimeUtils.currentTime()); partitionInfo.createRegionGroups(createRegionGroupsPlan); } GetRegionInfoListPlan regionReq = new GetRegionInfoListPlan(); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/CreateRegionGroupsProcedureTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/CreateRegionGroupsProcedureTest.java index 9cd9d41301c..15108131c61 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/CreateRegionGroupsProcedureTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/CreateRegionGroupsProcedureTest.java @@ -24,6 +24,7 @@ import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; +import org.apache.iotdb.commons.utils.CommonDateTimeUtils; import org.apache.iotdb.confignode.consensus.request.write.region.CreateRegionGroupsPlan; import org.apache.iotdb.confignode.procedure.impl.statemachine.CreateRegionGroupsProcedure; import org.apache.iotdb.confignode.procedure.store.ProcedureFactory; @@ -91,11 +92,13 @@ public class CreateRegionGroupsProcedureTest { assertEquals(failedRegions0, failedRegions1); CreateRegionGroupsPlan createRegionGroupsPlan = new CreateRegionGroupsPlan(); - createRegionGroupsPlan.addRegionGroup("root.sg0", dataRegionSet); - createRegionGroupsPlan.addRegionGroup("root.sg1", schemaRegionSet); + createRegionGroupsPlan.addRegionGroup( + "root.sg0", dataRegionSet, CommonDateTimeUtils.currentTime()); + createRegionGroupsPlan.addRegionGroup( + "root.sg1", schemaRegionSet, CommonDateTimeUtils.currentTime()); CreateRegionGroupsPlan persistPlan = new CreateRegionGroupsPlan(); - persistPlan.addRegionGroup("root.sg0", dataRegionSet); - persistPlan.addRegionGroup("root.sg1", schemaRegionSet); + persistPlan.addRegionGroup("root.sg0", dataRegionSet, CommonDateTimeUtils.currentTime()); + persistPlan.addRegionGroup("root.sg1", schemaRegionSet, CommonDateTimeUtils.currentTime()); CreateRegionGroupsProcedure procedure0 = new CreateRegionGroupsProcedure( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/audit/AuditLogger.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/audit/AuditLogger.java index 4db6c11ea33..aad36dbafb1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/audit/AuditLogger.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/audit/AuditLogger.java @@ -21,6 +21,7 @@ package org.apache.iotdb.db.audit; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.utils.CommonDateTimeUtils; import org.apache.iotdb.db.auth.AuthorityChecker; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; @@ -35,7 +36,6 @@ import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeDeviceP import org.apache.iotdb.db.queryengine.plan.statement.Statement; import org.apache.iotdb.db.queryengine.plan.statement.StatementType; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; -import org.apache.iotdb.db.utils.DateTimeUtils; import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.tsfile.common.conf.TSFileConfig; import org.apache.iotdb.tsfile.enums.TSDataType; @@ -85,7 +85,7 @@ public class AuditLogger { InsertRowStatement insertStatement = new InsertRowStatement(); insertStatement.setDevicePath( DEVICE_PATH_CACHE.getPartialPath(String.format(AUDIT_LOG_DEVICE, username))); - insertStatement.setTime(DateTimeUtils.currentTime()); + insertStatement.setTime(CommonDateTimeUtils.currentTime()); insertStatement.setMeasurements(new String[] {LOG, USERNAME, ADDRESS}); insertStatement.setAligned(false); insertStatement.setValues( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index f0ec678e894..7e4d509c207 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -799,9 +799,6 @@ public class IoTDBConfig { private float udfCollectorMemoryBudgetInMB = (float) (1.0 / 3 * udfMemoryBudgetInMB); - // time in nanosecond precision when starting up - private long startUpNanosecond = System.nanoTime(); - /** Unit: byte */ private int thriftMaxFrameSize = 536870912; @@ -2512,10 +2509,6 @@ public class IoTDBConfig { this.primitiveArraySize = primitiveArraySize; } - public long getStartUpNanosecond() { - return startUpNanosecond; - } - public int getThriftMaxFrameSize() { return thriftMaxFrameSize; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java index 108534f4bb7..4750d5e60af 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java @@ -29,6 +29,7 @@ import org.apache.iotdb.commons.cq.TimeoutPolicy; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.schema.filter.SchemaFilter; import org.apache.iotdb.commons.schema.filter.SchemaFilterFactory; +import org.apache.iotdb.commons.utils.CommonDateTimeUtils; import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; @@ -971,7 +972,8 @@ public class ASTVisitor extends IoTDBSqlParserBaseVisitor<Statement> { } if (ctx.BOUNDARY() != null) { - statement.setBoundaryTime(parseTimeValue(ctx.boundaryTime, DateTimeUtils.currentTime())); + statement.setBoundaryTime( + parseTimeValue(ctx.boundaryTime, CommonDateTimeUtils.currentTime())); } if (ctx.RANGE() != null) { @@ -1511,7 +1513,7 @@ public class ASTVisitor extends IoTDBSqlParserBaseVisitor<Statement> { */ private void parseTimeRangeForGroupByTime( IoTDBSqlParser.TimeRangeContext timeRange, GroupByTimeComponent groupByClauseComponent) { - long currentTime = DateTimeUtils.currentTime(); + long currentTime = CommonDateTimeUtils.currentTime(); long startTime = parseTimeValue(timeRange.timeValue(0), currentTime); long endTime = parseTimeValue(timeRange.timeValue(1), currentTime); groupByClauseComponent.setStartTime(startTime); @@ -1799,10 +1801,11 @@ public class ASTVisitor extends IoTDBSqlParserBaseVisitor<Statement> { throw new SemanticException("need timestamps when insert multi rows"); } valueList.add(insertMultiValues.get(i).timeValue().getText()); - timestamp = DateTimeUtils.currentTime(); + timestamp = CommonDateTimeUtils.currentTime(); } else { timestamp = - parseTimeValue(insertMultiValues.get(i).timeValue(), DateTimeUtils.currentTime()); + parseTimeValue( + insertMultiValues.get(i).timeValue(), CommonDateTimeUtils.currentTime()); TimestampPrecisionUtils.checkTimestampPrecision(timestamp); } } else { @@ -2027,7 +2030,7 @@ public class ASTVisitor extends IoTDBSqlParserBaseVisitor<Statement> { throw new SemanticException("input timestamp cannot be empty"); } if (timestampStr.equalsIgnoreCase(SqlConstant.NOW_FUNC)) { - return DateTimeUtils.currentTime(); + return CommonDateTimeUtils.currentTime(); } try { return DateTimeUtils.convertDatetimeStrToLong(timestampStr, zoneId); @@ -3650,7 +3653,7 @@ public class ASTVisitor extends IoTDBSqlParserBaseVisitor<Statement> { getRegionIdStatement.setDevice(ctx.device.getText()); } if (ctx.time != null) { - long timestamp = parseTimeValue(ctx.time, DateTimeUtils.currentTime()); + long timestamp = parseTimeValue(ctx.time, CommonDateTimeUtils.currentTime()); getRegionIdStatement.setTimeStamp(timestamp); } return getRegionIdStatement; @@ -3674,11 +3677,11 @@ public class ASTVisitor extends IoTDBSqlParserBaseVisitor<Statement> { getTimeSlotListStatement.setRegionId(Integer.parseInt(ctx.regionId.getText())); } if (ctx.startTime != null) { - long timestamp = parseTimeValue(ctx.startTime, DateTimeUtils.currentTime()); + long timestamp = parseTimeValue(ctx.startTime, CommonDateTimeUtils.currentTime()); getTimeSlotListStatement.setStartTime(timestamp); } if (ctx.endTime != null) { - long timestamp = parseTimeValue(ctx.endTime, DateTimeUtils.currentTime()); + long timestamp = parseTimeValue(ctx.endTime, CommonDateTimeUtils.currentTime()); getTimeSlotListStatement.setEndTime(timestamp); } return getTimeSlotListStatement; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/SeriesScanOptions.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/SeriesScanOptions.java index 82dd3ba5772..f2238efee2c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/SeriesScanOptions.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/SeriesScanOptions.java @@ -21,7 +21,7 @@ package org.apache.iotdb.db.queryengine.plan.planner.plan.parameter; import org.apache.iotdb.commons.path.AlignedPath; import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.db.utils.DateTimeUtils; +import org.apache.iotdb.commons.utils.CommonDateTimeUtils; import org.apache.iotdb.tsfile.read.filter.TimeFilter; import org.apache.iotdb.tsfile.read.filter.basic.Filter; import org.apache.iotdb.tsfile.read.filter.operator.AndFilter; @@ -100,9 +100,10 @@ public class SeriesScanOptions { public static Filter updateFilterUsingTTL(Filter filter, long dataTTL) { if (dataTTL != Long.MAX_VALUE) { if (filter != null) { - filter = new AndFilter(filter, TimeFilter.gtEq(DateTimeUtils.currentTime() - dataTTL)); + filter = + new AndFilter(filter, TimeFilter.gtEq(CommonDateTimeUtils.currentTime() - dataTTL)); } else { - filter = TimeFilter.gtEq(DateTimeUtils.currentTime() - dataTTL); + filter = TimeFilter.gtEq(CommonDateTimeUtils.currentTime() - dataTTL); } } return filter; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 3d63bc658ba..cdc2d1aa46a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -845,7 +845,8 @@ public class DataRegion implements IDataRegionForQuery { public void insert(InsertRowNode insertRowNode) throws WriteProcessException { // reject insertions that are out of ttl if (!isAlive(insertRowNode.getTime())) { - throw new OutOfTTLException(insertRowNode.getTime(), (DateTimeUtils.currentTime() - dataTTL)); + throw new OutOfTTLException( + insertRowNode.getTime(), (CommonDateTimeUtils.currentTime() - dataTTL)); } if (enableMemControl) { StorageEngine.blockInsertionIfReject(null); @@ -926,7 +927,8 @@ public class DataRegion implements IDataRegionForQuery { String.format( "Insertion time [%s] is less than ttl time bound [%s]", DateTimeUtils.convertLongToDate(currTime), - DateTimeUtils.convertLongToDate(DateTimeUtils.currentTime() - dataTTL))); + DateTimeUtils.convertLongToDate( + CommonDateTimeUtils.currentTime() - dataTTL))); loc++; noFailure = false; } else { @@ -937,7 +939,7 @@ public class DataRegion implements IDataRegionForQuery { if (loc == insertTabletNode.getRowCount()) { throw new OutOfTTLException( insertTabletNode.getTimes()[insertTabletNode.getTimes().length - 1], - (DateTimeUtils.currentTime() - dataTTL)); + (CommonDateTimeUtils.currentTime() - dataTTL)); } // before is first start point int before = loc; @@ -1011,7 +1013,7 @@ public class DataRegion implements IDataRegionForQuery { * @return whether the given time falls in ttl */ private boolean isAlive(long time) { - return dataTTL == Long.MAX_VALUE || (DateTimeUtils.currentTime() - time) <= dataTTL; + return dataTTL == Long.MAX_VALUE || (CommonDateTimeUtils.currentTime() - time) <= dataTTL; } /** @@ -1513,7 +1515,7 @@ public class DataRegion implements IDataRegionForQuery { logger.debug("{}: TTL not set, ignore the check", databaseName + "-" + dataRegionId); return; } - long ttlLowerBound = DateTimeUtils.currentTime() - dataTTL; + long ttlLowerBound = CommonDateTimeUtils.currentTime() - dataTTL; logger.debug( "{}: TTL removing files before {}", databaseName + "-" + dataRegionId, @@ -1764,7 +1766,7 @@ public class DataRegion implements IDataRegionForQuery { List<TsFileResource> tsfileResourcesForQuery = new ArrayList<>(); long timeLowerBound = - dataTTL != Long.MAX_VALUE ? DateTimeUtils.currentTime() - dataTTL : Long.MIN_VALUE; + dataTTL != Long.MAX_VALUE ? CommonDateTimeUtils.currentTime() - dataTTL : Long.MIN_VALUE; context.setQueryTimeLowerBound(timeLowerBound); for (TsFileResource tsFileResource : tsFileResources) { @@ -2742,7 +2744,8 @@ public class DataRegion implements IDataRegionForQuery { String.format( "Insertion time [%s] is less than ttl time bound [%s]", DateTimeUtils.convertLongToDate(insertRowNode.getTime()), - DateTimeUtils.convertLongToDate(DateTimeUtils.currentTime() - dataTTL)))); + DateTimeUtils.convertLongToDate( + CommonDateTimeUtils.currentTime() - dataTTL)))); continue; } // init map diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/MemTableFlushTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/MemTableFlushTask.java index 19450cbe9fc..1cf48033c63 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/MemTableFlushTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/MemTableFlushTask.java @@ -22,6 +22,7 @@ import org.apache.iotdb.commons.schema.SchemaConstant; import org.apache.iotdb.commons.service.metric.MetricService; import org.apache.iotdb.commons.service.metric.enums.Metric; import org.apache.iotdb.commons.service.metric.enums.Tag; +import org.apache.iotdb.commons.utils.CommonDateTimeUtils; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.service.metrics.WritingMetrics; @@ -31,7 +32,6 @@ import org.apache.iotdb.db.storageengine.dataregion.memtable.IMemTable; import org.apache.iotdb.db.storageengine.dataregion.memtable.IWritableMemChunk; import org.apache.iotdb.db.storageengine.dataregion.memtable.IWritableMemChunkGroup; import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo; -import org.apache.iotdb.db.utils.DateTimeUtils; import org.apache.iotdb.metrics.utils.MetricLevel; import org.apache.iotdb.tsfile.write.chunk.IChunkWriter; import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter; @@ -303,7 +303,7 @@ public class MemTableFlushTask { lastIndex = storageGroup.length(); } String storageGroupName = storageGroup.substring(0, lastIndex); - long currentTime = DateTimeUtils.currentTime(); + long currentTime = CommonDateTimeUtils.currentTime(); // compute the flush points long writeTime = flushPointsCache.compute( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java index 75f88327a16..0e228cbc7e0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java @@ -22,6 +22,7 @@ import org.apache.iotdb.commons.consensus.index.ProgressIndex; import org.apache.iotdb.commons.consensus.index.ProgressIndexType; import org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.utils.CommonDateTimeUtils; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; @@ -37,7 +38,6 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.FileTimeInd import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ITimeIndex; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.TimeIndexLevel; import org.apache.iotdb.db.storageengine.rescon.disk.TierManager; -import org.apache.iotdb.db.utils.DateTimeUtils; import org.apache.iotdb.tsfile.common.constant.TsFileConstant; import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata; import org.apache.iotdb.tsfile.file.metadata.ITimeSeriesMetadata; @@ -804,7 +804,7 @@ public class TsFileResource { /** @return whether the given time falls in ttl */ private boolean isAlive(long time, long dataTTL) { - return dataTTL == Long.MAX_VALUE || (DateTimeUtils.currentTime() - time) <= dataTTL; + return dataTTL == Long.MAX_VALUE || (CommonDateTimeUtils.currentTime() - time) <= dataTTL; } public void setProcessor(TsFileProcessor processor) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DateTimeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DateTimeUtils.java index 48befee79cd..278122486d5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DateTimeUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DateTimeUtils.java @@ -20,7 +20,6 @@ package org.apache.iotdb.db.utils; import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.utils.TestOnly; -import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.protocol.session.SessionManager; import org.apache.iotdb.tsfile.utils.TimeDuration; @@ -666,19 +665,6 @@ public class DateTimeUtils { } } - public static long currentTime() { - long startupNano = IoTDBDescriptor.getInstance().getConfig().getStartUpNanosecond(); - String timePrecision = CommonDescriptor.getInstance().getConfig().getTimestampPrecision(); - switch (timePrecision) { - case "ns": - return System.currentTimeMillis() * 1000_000 + (System.nanoTime() - startupNano) % 1000_000; - case "us": - return System.currentTimeMillis() * 1000 + (System.nanoTime() - startupNano) / 1000 % 1000; - default: - return System.currentTimeMillis(); - } - } - public static String convertLongToDate(long timestamp) { return convertLongToDate( timestamp, CommonDescriptor.getInstance().getConfig().getTimestampPrecision()); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java index e4abbc10f89..48a3a30dda9 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java @@ -209,6 +209,9 @@ public class CommonConfig { private long seriesLimitThreshold = -1; private long deviceLimitThreshold = -1; + // time in nanosecond precision when starting up + private final long startUpNanosecond = System.nanoTime(); + CommonConfig() { // Empty constructor } @@ -835,4 +838,8 @@ public class CommonConfig { public void setDeviceLimitThreshold(long deviceLimitThreshold) { this.deviceLimitThreshold = deviceLimitThreshold; } + + public long getStartUpNanosecond() { + return startUpNanosecond; + } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/CommonDateTimeUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/CommonDateTimeUtils.java index 3d65058abc9..2723c545265 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/CommonDateTimeUtils.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/CommonDateTimeUtils.java @@ -19,6 +19,8 @@ package org.apache.iotdb.commons.utils; +import org.apache.iotdb.commons.conf.CommonDescriptor; + public class CommonDateTimeUtils { public CommonDateTimeUtils() { @@ -39,4 +41,17 @@ public class CommonDateTimeUtils { } return result; } + + public static long currentTime() { + long startupNano = CommonDescriptor.getInstance().getConfig().getStartUpNanosecond(); + String timePrecision = CommonDescriptor.getInstance().getConfig().getTimestampPrecision(); + switch (timePrecision) { + case "ns": + return System.currentTimeMillis() * 1000_000 + (System.nanoTime() - startupNano) % 1000_000; + case "us": + return System.currentTimeMillis() * 1000 + (System.nanoTime() - startupNano) / 1000 % 1000; + default: + return System.currentTimeMillis(); + } + } }
