This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch dev/1.3
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/dev/1.3 by this push:
new 576918cbcfe [To dev/1.3] Use reference time position for
PartitionTableAutoCleaner
576918cbcfe is described below
commit 576918cbcfe34b5b67e942d7096c4abc358cd4ec
Author: Yongzao <[email protected]>
AuthorDate: Tue May 13 17:07:53 2025 +0800
[To dev/1.3] Use reference time position for PartitionTableAutoCleaner
---
.../partition/IoTDBPartitionTableAutoCleanIT.java | 7 +++--
....java => IoTDBPartitionTableAutoCleanUSIT.java} | 30 +++++++++++++---------
.../iotdb/confignode/manager/TTLManager.java | 9 +++++--
.../procedure/PartitionTableAutoCleaner.java | 29 ++++++++++++++++++---
.../iotdb/commons/utils/TimePartitionUtils.java | 13 ----------
5 files changed, 56 insertions(+), 32 deletions(-)
diff --git
a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java
b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java
index c62739cdc13..a7addb87c1d 100644
---
a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java
@@ -21,7 +21,6 @@ package org.apache.iotdb.confignode.it.partition;
import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot;
import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient;
-import org.apache.iotdb.commons.utils.TimePartitionUtils;
import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionReq;
import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionTableResp;
import org.apache.iotdb.it.env.EnvFactory;
@@ -52,7 +51,11 @@ public class IoTDBPartitionTableAutoCleanIT {
private static final long TEST_TTL_CHECK_INTERVAL = 5_000;
private static final TTimePartitionSlot TEST_CURRENT_TIME_SLOT =
- TimePartitionUtils.getCurrentTimePartitionSlot();
+ new TTimePartitionSlot()
+ .setStartTime(
+ System.currentTimeMillis()
+ / TEST_TIME_PARTITION_INTERVAL
+ * TEST_TIME_PARTITION_INTERVAL);
private static final long TEST_TTL = 7 * TEST_TIME_PARTITION_INTERVAL;
@Before
diff --git
a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java
b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanUSIT.java
similarity index 84%
copy from
integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java
copy to
integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanUSIT.java
index c62739cdc13..f8e94c28dcb 100644
---
a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanUSIT.java
@@ -21,7 +21,6 @@ package org.apache.iotdb.confignode.it.partition;
import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot;
import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient;
-import org.apache.iotdb.commons.utils.TimePartitionUtils;
import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionReq;
import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionTableResp;
import org.apache.iotdb.it.env.EnvFactory;
@@ -43,17 +42,22 @@ import java.util.concurrent.TimeUnit;
@RunWith(IoTDBTestRunner.class)
@Category({ClusterIT.class})
-public class IoTDBPartitionTableAutoCleanIT {
+public class IoTDBPartitionTableAutoCleanUSIT {
private static final String TREE_DATABASE_PREFIX = "root.db.g_";
private static final int TEST_REPLICATION_FACTOR = 1;
- private static final long TEST_TIME_PARTITION_INTERVAL = 604800000;
+ private static final long TEST_TIME_PARTITION_INTERVAL_IN_MS = 604800_000;
private static final long TEST_TTL_CHECK_INTERVAL = 5_000;
private static final TTimePartitionSlot TEST_CURRENT_TIME_SLOT =
- TimePartitionUtils.getCurrentTimePartitionSlot();
- private static final long TEST_TTL = 7 * TEST_TIME_PARTITION_INTERVAL;
+ new TTimePartitionSlot()
+ .setStartTime(
+ System.currentTimeMillis()
+ * 1000L
+ / TEST_TIME_PARTITION_INTERVAL_IN_MS
+ * TEST_TIME_PARTITION_INTERVAL_IN_MS);
+ private static final long TEST_TTL_IN_MS = 7 *
TEST_TIME_PARTITION_INTERVAL_IN_MS;
@Before
public void setUp() throws Exception {
@@ -62,8 +66,10 @@ public class IoTDBPartitionTableAutoCleanIT {
.getCommonConfig()
.setSchemaReplicationFactor(TEST_REPLICATION_FACTOR)
.setDataReplicationFactor(TEST_REPLICATION_FACTOR)
- .setTimePartitionInterval(TEST_TIME_PARTITION_INTERVAL)
- .setTTLCheckInterval(TEST_TTL_CHECK_INTERVAL);
+ .setTimePartitionInterval(TEST_TIME_PARTITION_INTERVAL_IN_MS)
+ .setTTLCheckInterval(TEST_TTL_CHECK_INTERVAL)
+ // Note that the time precision of IoTDB is us in this IT
+ .setTimestampPrecision("us");
// Init 1C1D environment
EnvFactory.getEnv().initClusterEnvironment(1, 1);
@@ -89,7 +95,7 @@ public class IoTDBPartitionTableAutoCleanIT {
statement.execute(
String.format(
"INSERT INTO %s(timestamp, s) VALUES (%d, %d)",
- databaseName, TEST_CURRENT_TIME_SLOT.getStartTime() - TEST_TTL
* 2, -1));
+ databaseName, TEST_CURRENT_TIME_SLOT.getStartTime() -
TEST_TTL_IN_MS * 2000, -1));
// Insert existed data
statement.execute(
String.format(
@@ -97,15 +103,15 @@ public class IoTDBPartitionTableAutoCleanIT {
databaseName, TEST_CURRENT_TIME_SLOT.getStartTime(), 1));
}
// Let db0.TTL > device.TTL, the valid TTL should be the bigger one
- statement.execute(String.format("SET TTL TO %s0 %d",
TREE_DATABASE_PREFIX, TEST_TTL));
+ statement.execute(String.format("SET TTL TO %s0 %d",
TREE_DATABASE_PREFIX, TEST_TTL_IN_MS));
statement.execute(String.format("SET TTL TO %s0.s %d",
TREE_DATABASE_PREFIX, 10));
// Let db1.TTL < device.TTL, the valid TTL should be the bigger one
statement.execute(String.format("SET TTL TO %s1 %d",
TREE_DATABASE_PREFIX, 10));
- statement.execute(String.format("SET TTL TO %s1.s %d",
TREE_DATABASE_PREFIX, TEST_TTL));
+ statement.execute(String.format("SET TTL TO %s1.s %d",
TREE_DATABASE_PREFIX, TEST_TTL_IN_MS));
// Set TTL to path db2.**
- statement.execute(String.format("SET TTL TO %s2.** %d",
TREE_DATABASE_PREFIX, TEST_TTL));
+ statement.execute(
+ String.format("SET TTL TO %s2.** %d", TREE_DATABASE_PREFIX,
TEST_TTL_IN_MS));
}
-
TDataPartitionReq req = new TDataPartitionReq();
for (int i = 0; i < 3; i++) {
req.putToPartitionSlotsMap(String.format("%s%d", TREE_DATABASE_PREFIX,
i), new TreeMap<>());
diff --git
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/TTLManager.java
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/TTLManager.java
index 55a61389126..b0c4dd5f296 100644
---
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/TTLManager.java
+++
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/TTLManager.java
@@ -22,6 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.commons.utils.CommonDateTimeUtils;
import org.apache.iotdb.commons.utils.PathUtils;
import org.apache.iotdb.confignode.consensus.request.read.ttl.ShowTTLPlan;
import
org.apache.iotdb.confignode.consensus.request.write.database.DatabaseSchemaPlan;
@@ -135,8 +136,12 @@ public class TTLManager {
* @return the maximum ttl of the subtree of the corresponding database.
return NULL_TTL if the
* TTL is not set or the database does not exist.
*/
- public long getDatabaseMaxTTL(String database) {
- return ttlInfo.getDatabaseMaxTTL(database);
+ public long getDatabaseMaxTTL(final String database) {
+ final long ttl = ttlInfo.getDatabaseMaxTTL(database);
+ return ttl == Long.MAX_VALUE || ttl < 0
+ ? ttl
+ : CommonDateTimeUtils.convertMilliTimeWithPrecision(
+ ttl,
CommonDescriptor.getInstance().getConfig().getTimestampPrecision());
}
/** Only used for upgrading from old database-level ttl to device-level ttl.
*/
diff --git
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/PartitionTableAutoCleaner.java
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/PartitionTableAutoCleaner.java
index ce5d07276db..8a08e5efe99 100644
---
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/PartitionTableAutoCleaner.java
+++
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/PartitionTableAutoCleaner.java
@@ -22,7 +22,6 @@ package org.apache.iotdb.confignode.procedure;
import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot;
import org.apache.iotdb.commons.conf.CommonConfig;
import org.apache.iotdb.commons.conf.CommonDescriptor;
-import org.apache.iotdb.commons.utils.TimePartitionUtils;
import
org.apache.iotdb.confignode.consensus.request.write.partition.AutoCleanPartitionTablePlan;
import org.apache.iotdb.confignode.manager.ConfigManager;
import org.apache.iotdb.consensus.exception.ConsensusException;
@@ -42,6 +41,10 @@ public class PartitionTableAutoCleaner<Env> extends
InternalProcedure<Env> {
private static final Logger LOGGER =
LoggerFactory.getLogger(PartitionTableAutoCleaner.class);
private static final CommonConfig COMMON_CONFIG =
CommonDescriptor.getInstance().getConfig();
+
+ private static final String timestampPrecision =
+ CommonDescriptor.getInstance().getConfig().getTimestampPrecision();
+
private final ConfigManager configManager;
public PartitionTableAutoCleaner(ConfigManager configManager) {
@@ -59,6 +62,12 @@ public class PartitionTableAutoCleaner<Env> extends
InternalProcedure<Env> {
for (String database : databases) {
long databaseTTL =
configManager.getTTLManager().getDatabaseMaxTTL(database);
databaseTTLMap.put(database, databaseTTL);
+ }
+ LOGGER.info(
+ "[PartitionTableCleaner] Periodically activate
PartitionTableAutoCleaner, databaseTTL: {}",
+ databaseTTLMap);
+ for (String database : databases) {
+ long databaseTTL = databaseTTLMap.get(database);
if (!configManager.getPartitionManager().isDatabaseExist(database)
|| databaseTTL < 0
|| databaseTTL == Long.MAX_VALUE) {
@@ -71,8 +80,7 @@ public class PartitionTableAutoCleaner<Env> extends
InternalProcedure<Env> {
"[PartitionTableCleaner] Periodically activate
PartitionTableAutoCleaner for: {}",
databaseTTLMap);
// Only clean the partition table when necessary
- TTimePartitionSlot currentTimePartitionSlot =
- TimePartitionUtils.getCurrentTimePartitionSlot();
+ TTimePartitionSlot currentTimePartitionSlot =
getCurrentTimePartitionSlot();
try {
configManager
.getConsensusManager()
@@ -82,4 +90,19 @@ public class PartitionTableAutoCleaner<Env> extends
InternalProcedure<Env> {
}
}
}
+
+ /**
+ * @return The time partition slot corresponding to current timestamp. Note
that we do not shift
+ * the start time to the correct starting point, since this interface
only constructs a time
+ * reference position for the partition table cleaner.
+ */
+ private static TTimePartitionSlot getCurrentTimePartitionSlot() {
+ if ("ms".equals(timestampPrecision)) {
+ return new TTimePartitionSlot(System.currentTimeMillis());
+ } else if ("us".equals(timestampPrecision)) {
+ return new TTimePartitionSlot(System.currentTimeMillis() * 1000);
+ } else {
+ return new TTimePartitionSlot(System.currentTimeMillis() * 1000_000);
+ }
+ }
}
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/TimePartitionUtils.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/TimePartitionUtils.java
index ed01e8a5f3f..7b331fddaac 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/TimePartitionUtils.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/TimePartitionUtils.java
@@ -34,9 +34,6 @@ public class TimePartitionUtils {
private static long timePartitionOrigin =
CommonDescriptor.getInstance().getConfig().getTimePartitionOrigin();
- private static String timestampPrecision =
- CommonDescriptor.getInstance().getConfig().getTimestampPrecision();
-
/** Time range for dividing database, the time unit is the same with IoTDB's
TimestampPrecision */
private static long timePartitionInterval =
CommonDescriptor.getInstance().getConfig().getTimePartitionInterval();
@@ -74,16 +71,6 @@ public class TimePartitionUtils {
}
}
- public static TTimePartitionSlot getCurrentTimePartitionSlot() {
- if ("ms".equals(timestampPrecision)) {
- return getTimePartitionSlot(System.currentTimeMillis());
- } else if ("us".equals(timestampPrecision)) {
- return getTimePartitionSlot(System.currentTimeMillis() * 1000);
- } else {
- return getTimePartitionSlot(System.currentTimeMillis() * 1000_000);
- }
- }
-
public static TTimePartitionSlot getTimePartitionSlot(long time) {
TTimePartitionSlot timePartitionSlot = new TTimePartitionSlot();
timePartitionSlot.setStartTime(getTimePartitionLowerBound(time));
