This is an automated email from the ASF dual-hosted git repository.
jt2594838 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new f3e161218d6 Fix downgraded TsFileResource serialization (#18155)
f3e161218d6 is described below
commit f3e161218d68c090c4802bfbf74c152e57be26a3
Author: Jiang Tian <[email protected]>
AuthorDate: Thu Jul 9 09:56:51 2026 +0800
Fix downgraded TsFileResource serialization (#18155)
* Fix downgraded resource may not serialize normally
* Revert FileIndex serialization
---
.../IoTDBRegionMigrateNormalITForIoTV2BatchIT.java | 34 ++++++++++++++++++
.../dataregion/tsfile/TsFileResource.java | 41 ++++++++++++++--------
.../dataregion/tsfile/TsFileResourceTest.java | 19 ++++++++--
3 files changed, 78 insertions(+), 16 deletions(-)
diff --git
a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/commit/batch/IoTDBRegionMigrateNormalITForIoTV2BatchIT.java
b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/commit/batch/IoTDBRegionMigrateNormalITForIoTV2BatchIT.java
index b763b424002..23cd7236ef2 100644
---
a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/commit/batch/IoTDBRegionMigrateNormalITForIoTV2BatchIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/commit/batch/IoTDBRegionMigrateNormalITForIoTV2BatchIT.java
@@ -21,13 +21,21 @@ package
org.apache.iotdb.confignode.it.regionmigration.pass.commit.batch;
import org.apache.iotdb.commons.utils.KillPoint.KillNode;
import
org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionOperationReliabilityITFramework;
+import org.apache.iotdb.it.env.EnvFactory;
import org.apache.iotdb.it.framework.IoTDBTestRunner;
import org.apache.iotdb.itbase.category.ClusterIT;
+import org.junit.Assert;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.Statement;
+
+import static org.apache.iotdb.util.MagicUtils.makeItCloseQuietly;
+
@Category({ClusterIT.class})
@RunWith(IoTDBTestRunner.class)
public class IoTDBRegionMigrateNormalITForIoTV2BatchIT
@@ -41,4 +49,30 @@ public class IoTDBRegionMigrateNormalITForIoTV2BatchIT
public void normal3C3DTest() throws Exception {
successTest(2, 3, 3, 3, noKillPoints(), noKillPoints(),
KillNode.ALL_NODES);
}
+
+ @Test
+ public void migrateRegionWithDegradedTimeIndexTest() throws Exception {
+ // set TsFileResource memory to 0 to trigger degrading
+
EnvFactory.getEnv().getConfig().getCommonConfig().setQueryMemoryProportion("1:1:1:1:1:1:0");
+
+ successTest(1, 1, 1, 2, noKillPoints(), noKillPoints(),
KillNode.ALL_NODES);
+
+ try (final Connection connection =
makeItCloseQuietly(EnvFactory.getEnv().getConnection());
+ final Statement statement =
makeItCloseQuietly(connection.createStatement())) {
+ assertCounts(statement, 1, 1);
+ statement.execute("INSERT INTO root.sg.d1(timestamp,speed,temperature)
values(101, 3, 4)");
+ assertCounts(statement, 2, 2);
+ }
+ }
+
+ private static void assertCounts(
+ final Statement statement, final long expectedSpeedCount, final long
expectedTemperatureCount)
+ throws Exception {
+ try (final ResultSet resultSet =
+ statement.executeQuery("select count(speed), count(temperature) from
root.sg.d1")) {
+ Assert.assertTrue(resultSet.next());
+ Assert.assertEquals(expectedSpeedCount, resultSet.getLong(1));
+ Assert.assertEquals(expectedTemperatureCount, resultSet.getLong(2));
+ }
+ }
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
index 80476e465ec..2b2a292f247 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
@@ -283,7 +283,7 @@ public class TsFileResource implements PersistentResource,
Cloneable {
private void serializeTo(BufferedOutputStream outputStream) throws
IOException {
ReadWriteIOUtils.write(VERSION_NUMBER, outputStream);
- timeIndex.serialize(outputStream);
+ getTimeIndexForSerialization().serialize(outputStream);
ReadWriteIOUtils.write(maxPlanIndex, outputStream);
ReadWriteIOUtils.write(minPlanIndex, outputStream);
@@ -317,6 +317,14 @@ public class TsFileResource implements PersistentResource,
Cloneable {
ReadWriteIOUtils.write(isGeneratedByPipe, outputStream);
}
+ private ITimeIndex getTimeIndexForSerialization() throws IOException {
+ if (!(timeIndex instanceof FileTimeIndex) || !resourceFileExists()) {
+ return timeIndex;
+ }
+
+ return buildDeviceTimeIndex();
+ }
+
/** deserialize from disk */
public void deserialize() throws IOException {
try (InputStream inputStream = fsFactory.getBufferedInputStream(file +
RESOURCE_SUFFIX)) {
@@ -694,20 +702,13 @@ public class TsFileResource implements
PersistentResource, Cloneable {
throws IOException {
readLock();
try {
- if (!resourceFileExists()) {
- throw new IOException(StorageEngineMessages.RESOURCE_FILE_NOT_FOUND);
- }
- try (InputStream inputStream =
- FSFactoryProducer.getFSFactory()
- .getBufferedInputStream(file.getPath() + RESOURCE_SUFFIX)) {
- ReadWriteIOUtils.readByte(inputStream);
- ITimeIndex timeIndexFromResourceFile =
- ITimeIndex.createTimeIndex(inputStream, deserializer);
- if (!(timeIndexFromResourceFile instanceof ArrayDeviceTimeIndex)) {
- throw new IOException(
- StorageEngineMessages.CANNOT_BUILD_DEVICE_TIME_INDEX +
file.getPath());
+ try {
+ ITimeIndex timeIndexFromResourceFile =
deserializeTimeIndexFromResourceFile(deserializer);
+ if (timeIndexFromResourceFile instanceof ArrayDeviceTimeIndex) {
+ return (ArrayDeviceTimeIndex) timeIndexFromResourceFile;
}
- return (ArrayDeviceTimeIndex) timeIndexFromResourceFile;
+ throw new IOException(
+ StorageEngineMessages.CANNOT_BUILD_DEVICE_TIME_INDEX +
file.getPath());
} catch (Exception e) {
throw new IOException(
String.format(
@@ -721,6 +722,18 @@ public class TsFileResource implements PersistentResource,
Cloneable {
}
}
+ private ITimeIndex
deserializeTimeIndexFromResourceFile(IDeviceID.Deserializer deserializer)
+ throws IOException {
+ if (!resourceFileExists()) {
+ throw new IOException(StorageEngineMessages.RESOURCE_FILE_NOT_FOUND);
+ }
+ try (InputStream inputStream =
+ FSFactoryProducer.getFSFactory().getBufferedInputStream(file.getPath()
+ RESOURCE_SUFFIX)) {
+ ReadWriteIOUtils.readByte(inputStream);
+ return ITimeIndex.createTimeIndex(inputStream, deserializer);
+ }
+ }
+
public ArrayDeviceTimeIndex buildDeviceTimeIndex() throws IOException {
return buildDeviceTimeIndex(IDeviceID.Deserializer.DEFAULT_DESERIALIZER);
}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResourceTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResourceTest.java
index d637450eee3..ba81d806287 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResourceTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResourceTest.java
@@ -93,7 +93,7 @@ public class TsFileResourceTest {
if (file.exists()) {
FileUtils.delete(file);
}
- File resourceFile = new File(file.getName() +
TsFileResource.RESOURCE_SUFFIX);
+ File resourceFile = new File(file.getPath() +
TsFileResource.RESOURCE_SUFFIX);
if (resourceFile.exists()) {
FileUtils.delete(resourceFile);
}
@@ -108,7 +108,22 @@ public class TsFileResourceTest {
}
@Test
- public void testDegradeAndFileTimeIndex() {
+ public void testSerializeDegradedTimeIndex() throws IOException {
+ tsFileResource.serialize();
+ tsFileResource.degradeTimeIndex();
+
+ tsFileResource.serialize();
+
+ TsFileResource derTsFileResource = new TsFileResource(file);
+ derTsFileResource.deserialize();
+ Assert.assertEquals(
+ ITimeIndex.ARRAY_DEVICE_TIME_INDEX_TYPE,
derTsFileResource.getTimeIndexType());
+ Assert.assertEquals(deviceToIndex.keySet(),
derTsFileResource.getDevices());
+ }
+
+ @Test
+ public void testDegradeAndFileTimeIndex() throws IOException {
+ tsFileResource.serialize();
Assert.assertEquals(ITimeIndex.ARRAY_DEVICE_TIME_INDEX_TYPE,
tsFileResource.getTimeIndexType());
tsFileResource.degradeTimeIndex();
Assert.assertEquals(ITimeIndex.FILE_TIME_INDEX_TYPE,
tsFileResource.getTimeIndexType());
