This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new d4818124bc1 Use reference time position for PartitionTableAutoCleaner
d4818124bc1 is described below
commit d4818124bc1a6bd5ebf932396458ff608122ce01
Author: Yongzao <[email protected]>
AuthorDate: Tue May 13 17:17:04 2025 +0800
Use reference time position for PartitionTableAutoCleaner
---
.../partition/IoTDBPartitionTableAutoCleanIT.java | 7 +++--
....java => IoTDBPartitionTableAutoCleanUSIT.java} | 34 +++++++++++++---------
.../procedure/PartitionTableAutoCleaner.java | 23 +++++++++++++--
.../iotdb/commons/utils/TimePartitionUtils.java | 13 ---------
4 files changed, 45 insertions(+), 32 deletions(-)
diff --git
a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java
b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java
index eb6cdbddb2a..3f3596746f0 100644
---
a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java
@@ -21,7 +21,6 @@ package org.apache.iotdb.confignode.it.partition;
import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot;
import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient;
-import org.apache.iotdb.commons.utils.TimePartitionUtils;
import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionReq;
import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionTableResp;
import org.apache.iotdb.it.env.EnvFactory;
@@ -54,7 +53,11 @@ public class IoTDBPartitionTableAutoCleanIT {
private static final long TEST_TTL_CHECK_INTERVAL = 5_000;
private static final TTimePartitionSlot TEST_CURRENT_TIME_SLOT =
- TimePartitionUtils.getCurrentTimePartitionSlot();
+ new TTimePartitionSlot()
+ .setStartTime(
+ System.currentTimeMillis()
+ / TEST_TIME_PARTITION_INTERVAL
+ * TEST_TIME_PARTITION_INTERVAL);
private static final long TEST_TTL = 7 * TEST_TIME_PARTITION_INTERVAL;
@Before
diff --git
a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java
b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanUSIT.java
similarity index 87%
copy from
integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java
copy to
integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanUSIT.java
index eb6cdbddb2a..69ab3a03d28 100644
---
a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanUSIT.java
@@ -21,7 +21,6 @@ package org.apache.iotdb.confignode.it.partition;
import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot;
import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient;
-import org.apache.iotdb.commons.utils.TimePartitionUtils;
import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionReq;
import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionTableResp;
import org.apache.iotdb.it.env.EnvFactory;
@@ -44,18 +43,23 @@ import java.util.concurrent.TimeUnit;
@RunWith(IoTDBTestRunner.class)
@Category({ClusterIT.class})
-public class IoTDBPartitionTableAutoCleanIT {
+public class IoTDBPartitionTableAutoCleanUSIT {
private static final String TREE_DATABASE_PREFIX = "root.db.g_";
private static final String TABLE_DATABASE_PREFIX = "database_";
private static final int TEST_REPLICATION_FACTOR = 1;
- private static final long TEST_TIME_PARTITION_INTERVAL = 604800000;
+ private static final long TEST_TIME_PARTITION_INTERVAL_IN_MS = 604800_000;
private static final long TEST_TTL_CHECK_INTERVAL = 5_000;
private static final TTimePartitionSlot TEST_CURRENT_TIME_SLOT =
- TimePartitionUtils.getCurrentTimePartitionSlot();
- private static final long TEST_TTL = 7 * TEST_TIME_PARTITION_INTERVAL;
+ new TTimePartitionSlot()
+ .setStartTime(
+ System.currentTimeMillis()
+ * 1000L
+ / TEST_TIME_PARTITION_INTERVAL_IN_MS
+ * TEST_TIME_PARTITION_INTERVAL_IN_MS);
+ private static final long TEST_TTL_IN_MS = 7 *
TEST_TIME_PARTITION_INTERVAL_IN_MS;
@Before
public void setUp() throws Exception {
@@ -64,8 +68,10 @@ public class IoTDBPartitionTableAutoCleanIT {
.getCommonConfig()
.setSchemaReplicationFactor(TEST_REPLICATION_FACTOR)
.setDataReplicationFactor(TEST_REPLICATION_FACTOR)
- .setTimePartitionInterval(TEST_TIME_PARTITION_INTERVAL)
- .setTTLCheckInterval(TEST_TTL_CHECK_INTERVAL);
+ .setTimePartitionInterval(TEST_TIME_PARTITION_INTERVAL_IN_MS)
+ .setTTLCheckInterval(TEST_TTL_CHECK_INTERVAL)
+ // Note that the time precision of IoTDB is us in this IT
+ .setTimestampPrecision("us");
// Init 1C1D environment
EnvFactory.getEnv().initClusterEnvironment(1, 1);
@@ -91,7 +97,7 @@ public class IoTDBPartitionTableAutoCleanIT {
statement.execute(
String.format(
"INSERT INTO %s(timestamp, s) VALUES (%d, %d)",
- databaseName, TEST_CURRENT_TIME_SLOT.getStartTime() - TEST_TTL
* 2, -1));
+ databaseName, TEST_CURRENT_TIME_SLOT.getStartTime() -
TEST_TTL_IN_MS * 2000, -1));
// Insert existed data
statement.execute(
String.format(
@@ -99,15 +105,15 @@ public class IoTDBPartitionTableAutoCleanIT {
databaseName, TEST_CURRENT_TIME_SLOT.getStartTime(), 1));
}
// Let db0.TTL > device.TTL, the valid TTL should be the bigger one
- statement.execute(String.format("SET TTL TO %s0 %d",
TREE_DATABASE_PREFIX, TEST_TTL));
+ statement.execute(String.format("SET TTL TO %s0 %d",
TREE_DATABASE_PREFIX, TEST_TTL_IN_MS));
statement.execute(String.format("SET TTL TO %s0.s %d",
TREE_DATABASE_PREFIX, 10));
// Let db1.TTL < device.TTL, the valid TTL should be the bigger one
statement.execute(String.format("SET TTL TO %s1 %d",
TREE_DATABASE_PREFIX, 10));
- statement.execute(String.format("SET TTL TO %s1.s %d",
TREE_DATABASE_PREFIX, TEST_TTL));
+ statement.execute(String.format("SET TTL TO %s1.s %d",
TREE_DATABASE_PREFIX, TEST_TTL_IN_MS));
// Set TTL to path db2.**
- statement.execute(String.format("SET TTL TO %s2.** %d",
TREE_DATABASE_PREFIX, TEST_TTL));
+ statement.execute(
+ String.format("SET TTL TO %s2.** %d", TREE_DATABASE_PREFIX,
TEST_TTL_IN_MS));
}
-
TDataPartitionReq req = new TDataPartitionReq();
for (int i = 0; i < 3; i++) {
req.putToPartitionSlotsMap(String.format("%s%d", TREE_DATABASE_PREFIX,
i), new TreeMap<>());
@@ -146,13 +152,13 @@ public class IoTDBPartitionTableAutoCleanIT {
statement.execute(
String.format(
"INSERT INTO tb(time, s) VALUES (%d, %d)",
- TEST_CURRENT_TIME_SLOT.getStartTime() - TEST_TTL * 2, -1));
+ TEST_CURRENT_TIME_SLOT.getStartTime() - TEST_TTL_IN_MS * 2000,
-1));
// Insert existed data
statement.execute(
String.format(
"INSERT INTO tb(time, s) VALUES (%d, %d)",
TEST_CURRENT_TIME_SLOT.getStartTime(), 1));
statement.execute(String.format("USE %s", TABLE_DATABASE_PREFIX));
- statement.execute(String.format("ALTER TABLE tb SET PROPERTIES TTL=%d",
TEST_TTL));
+ statement.execute(String.format("ALTER TABLE tb SET PROPERTIES TTL=%d",
TEST_TTL_IN_MS));
}
TDataPartitionReq req = new TDataPartitionReq();
diff --git
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/PartitionTableAutoCleaner.java
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/PartitionTableAutoCleaner.java
index a0baa3a77c7..2ab15283207 100644
---
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/PartitionTableAutoCleaner.java
+++
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/PartitionTableAutoCleaner.java
@@ -23,7 +23,6 @@ import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot;
import org.apache.iotdb.commons.conf.CommonConfig;
import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.utils.PathUtils;
-import org.apache.iotdb.commons.utils.TimePartitionUtils;
import
org.apache.iotdb.confignode.consensus.request.write.partition.AutoCleanPartitionTablePlan;
import org.apache.iotdb.confignode.manager.ConfigManager;
import org.apache.iotdb.consensus.exception.ConsensusException;
@@ -43,6 +42,10 @@ public class PartitionTableAutoCleaner<Env> extends
InternalProcedure<Env> {
private static final Logger LOGGER =
LoggerFactory.getLogger(PartitionTableAutoCleaner.class);
private static final CommonConfig COMMON_CONFIG =
CommonDescriptor.getInstance().getConfig();
+
+ private static final String timestampPrecision =
+ CommonDescriptor.getInstance().getConfig().getTimestampPrecision();
+
private final ConfigManager configManager;
public PartitionTableAutoCleaner(ConfigManager configManager) {
@@ -81,8 +84,7 @@ public class PartitionTableAutoCleaner<Env> extends
InternalProcedure<Env> {
"[PartitionTableCleaner] Periodically activate
PartitionTableAutoCleaner for: {}",
databaseTTLMap);
// Only clean the partition table when necessary
- TTimePartitionSlot currentTimePartitionSlot =
- TimePartitionUtils.getCurrentTimePartitionSlot();
+ TTimePartitionSlot currentTimePartitionSlot =
getCurrentTimePartitionSlot();
try {
configManager
.getConsensusManager()
@@ -92,4 +94,19 @@ public class PartitionTableAutoCleaner<Env> extends
InternalProcedure<Env> {
}
}
}
+
+ /**
+ * @return The time partition slot corresponding to current timestamp. Note
that we do not shift
+ * the start time to the correct starting point, since this interface
only constructs a time
+ * reference position for the partition table cleaner.
+ */
+ private static TTimePartitionSlot getCurrentTimePartitionSlot() {
+ if ("ms".equals(timestampPrecision)) {
+ return new TTimePartitionSlot(System.currentTimeMillis());
+ } else if ("us".equals(timestampPrecision)) {
+ return new TTimePartitionSlot(System.currentTimeMillis() * 1000);
+ } else {
+ return new TTimePartitionSlot(System.currentTimeMillis() * 1000_000);
+ }
+ }
}
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/TimePartitionUtils.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/TimePartitionUtils.java
index 058f7433929..eb53cdb2798 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/TimePartitionUtils.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/TimePartitionUtils.java
@@ -34,9 +34,6 @@ public class TimePartitionUtils {
private static long timePartitionOrigin =
CommonDescriptor.getInstance().getConfig().getTimePartitionOrigin();
- private static String timestampPrecision =
- CommonDescriptor.getInstance().getConfig().getTimestampPrecision();
-
/** Time range for dividing database, the time unit is the same with IoTDB's
TimestampPrecision */
private static long timePartitionInterval =
CommonDescriptor.getInstance().getConfig().getTimePartitionInterval();
@@ -74,16 +71,6 @@ public class TimePartitionUtils {
}
}
- public static TTimePartitionSlot getCurrentTimePartitionSlot() {
- if ("ms".equals(timestampPrecision)) {
- return getTimePartitionSlot(System.currentTimeMillis());
- } else if ("us".equals(timestampPrecision)) {
- return getTimePartitionSlot(System.currentTimeMillis() * 1000);
- } else {
- return getTimePartitionSlot(System.currentTimeMillis() * 1000_000);
- }
- }
-
public static TTimePartitionSlot getTimePartitionSlot(long time) {
TTimePartitionSlot timePartitionSlot = new TTimePartitionSlot();
timePartitionSlot.setStartTime(getTimePartitionLowerBound(time));
