This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 3143fd53d2f Register TableSchema for sql insertion & Fix recovering V3
TsFile (#13682)
3143fd53d2f is described below
commit 3143fd53d2fe964efd34f83cdc7a0091e5ac64e0
Author: Jiang Tian <[email protected]>
AuthorDate: Thu Oct 10 12:12:25 2024 +0800
Register TableSchema for sql insertion & Fix recovering V3 TsFile (#13682)
Co-authored-by: Haonan <[email protected]>
---
.../plan/node/write/RelationalInsertRowsNode.java | 5 ++-
.../db/storageengine/dataregion/DataRegion.java | 5 ++-
.../file/AbstractTsFileRecoverPerformer.java | 40 +++++++++-------------
.../dataregion/wal/DisableWALTest.java | 1 +
.../wal/recover/WALRecoverManagerTest.java | 17 +++++++--
.../file/SealedTsFileRecoverPerformerTest.java | 1 +
6 files changed, 41 insertions(+), 28 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowsNode.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowsNode.java
index 768bb920905..71faebec3ca 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowsNode.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowsNode.java
@@ -151,7 +151,10 @@ public class RelationalInsertRowsNode extends
InsertRowsNode {
}
public String getTableName() {
- return targetPath.getFullPath();
+ if (targetPath != null) {
+ return targetPath.getFullPath();
+ }
+ return getInsertRowNodeList().get(0).getTableName();
}
@Override
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
index 1a98874f5fb..2abcfe8ebc3 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
@@ -1394,6 +1394,8 @@ public class DataRegion implements IDataRegionForQuery {
PERFORMANCE_OVERVIEW_METRICS.recordScheduleMemoryBlockCost(costsForMetrics[1]);
PERFORMANCE_OVERVIEW_METRICS.recordScheduleWalCost(costsForMetrics[2]);
PERFORMANCE_OVERVIEW_METRICS.recordScheduleMemTableCost(costsForMetrics[3]);
+ // register TableSchema (and maybe more) for table insertion
+ registerToTsFile(insertRowNode, tsFileProcessor);
return tsFileProcessor;
}
@@ -1478,7 +1480,8 @@ public class DataRegion implements IDataRegionForQuery {
RpcUtils.getStatus(e.getErrorCode(), e.getMessage()));
}
executedInsertRowNodeList.addAll(subInsertRowsNode.getInsertRowNodeList());
-
+ // register TableSchema (and maybe more) for table insertion
+ registerToTsFile(subInsertRowsNode, tsFileProcessor);
// check memtable size and may asyncTryToFlush the work memtable
if (entry.getKey().shouldFlush()) {
fileFlushPolicy.apply(this, tsFileProcessor,
tsFileProcessor.isSequence());
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/AbstractTsFileRecoverPerformer.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/AbstractTsFileRecoverPerformer.java
index cec93e0ead4..73621a5208e 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/AbstractTsFileRecoverPerformer.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/AbstractTsFileRecoverPerformer.java
@@ -25,6 +25,7 @@ import
org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.iotdb.db.storageengine.dataregion.utils.TsFileResourceUtils;
import org.apache.commons.io.FileUtils;
+import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.exception.NotCompatibleTsFileException;
import org.apache.tsfile.read.TsFileSequenceReader;
import org.apache.tsfile.write.writer.RestorableTsFileIOWriter;
@@ -35,7 +36,6 @@ import org.slf4j.LoggerFactory;
import java.io.Closeable;
import java.io.File;
import java.io.IOException;
-import java.nio.file.Files;
/** This class is used to help recover TsFile. */
public abstract class AbstractTsFileRecoverPerformer implements Closeable {
@@ -79,30 +79,24 @@ public abstract class AbstractTsFileRecoverPerformer
implements Closeable {
return;
}
- // try to remove corrupted part of the TsFile
- try {
- writer = new RestorableTsFileIOWriter(tsFile);
- } catch (NotCompatibleTsFileException e) {
- boolean result = tsFile.delete();
- logger.warn(
- "TsFile {} is incompatible. Try to delete it and delete result is
{}", tsFile, result);
- // if the broken TsFile is v3, we can recover the all data from wal
- // to support it, we can regenerate an empty file here
- Files.createFile(tsFile.toPath());
- writer = new RestorableTsFileIOWriter(tsFile);
- throw new DataRegionException(e);
- } catch (IOException e) {
- throw new DataRegionException(e);
- }
+ writer = new RestorableTsFileIOWriter(tsFile);
+ if (writer.hasCrashed()) {
+ byte versionNumber;
+ try (TsFileSequenceReader sequenceReader =
+ new TsFileSequenceReader(tsFile.getAbsolutePath())) {
+ versionNumber = sequenceReader.readVersionNumber();
+ } catch (NotCompatibleTsFileException e) {
+ versionNumber = -1;
+ }
- // reconstruct .resource file when TsFile is complete
- if (!writer.hasCrashed()) {
- try {
- reconstructResourceFile();
- } catch (IOException e) {
- throw new DataRegionException(
- "Failed recover the resource file: " + tsFile +
TsFileResource.RESOURCE_SUFFIX + e);
+ if (versionNumber != TSFileConfig.VERSION_NUMBER) {
+ // cannot rewrite a file with V3 header, delete it first
+ writer.close();
+ tsFile.delete();
+ writer = new RestorableTsFileIOWriter(tsFile);
}
+ } else {
+ reconstructResourceFile();
}
}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/DisableWALTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/DisableWALTest.java
index 35898df6b2a..abe7f3d8161 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/DisableWALTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/DisableWALTest.java
@@ -43,6 +43,7 @@ public class DisableWALTest {
@After
public void tearDown() throws Exception {
config.setWalMode(prevMode);
+ WALManager.getInstance().clear();
}
@Test
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverManagerTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverManagerTest.java
index 0dc14673b49..51f17facd91 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverManagerTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverManagerTest.java
@@ -67,7 +67,6 @@ import org.apache.tsfile.write.schema.MeasurementSchema;
import org.awaitility.Awaitility;
import org.junit.After;
import org.junit.Before;
-import org.junit.Ignore;
import org.junit.Test;
import java.io.File;
@@ -115,9 +114,11 @@ public class WALRecoverManagerTest {
private TsFileResource tsFileWithoutWALResource;
private long originWALThreshold =
IoTDBDescriptor.getInstance().getConfig().getWalFileSizeThresholdInByte();
+ private boolean bufferClosed = false;
@Before
public void setUp() throws Exception {
+ WALRecoverManager.getInstance().clear();
IoTDBDescriptor.getInstance().getConfig().setWalFileSizeThresholdInByte(1
* 1024 * 1024);
EnvironmentUtils.cleanDir(new File(FILE_WITH_WAL_NAME).getParent());
EnvironmentUtils.envSetUp();
@@ -125,6 +126,7 @@ public class WALRecoverManagerTest {
config.setWalMode(WALMode.SYNC);
walBuffer = new WALBuffer(WAL_NODE_IDENTIFIER, WAL_NODE_FOLDER);
checkpointManager = walBuffer.getCheckpointManager();
+ bufferClosed = false;
}
@After
@@ -137,7 +139,9 @@ public class WALRecoverManagerTest {
tsFileWithoutWALResource.close();
}
checkpointManager.close();
- walBuffer.close();
+ if (!bufferClosed) {
+ walBuffer.close();
+ }
config.setWalMode(prevMode);
EnvironmentUtils.cleanDir(new File(FILE_WITH_WAL_NAME).getParent());
EnvironmentUtils.cleanDir(new File(FILE_WITHOUT_WAL_NAME).getParent());
@@ -148,6 +152,8 @@ public class WALRecoverManagerTest {
public void testNormalProcedure() throws Exception {
prepareCheckpointAndWALFileForNormal();
WALRecoverManager.getInstance().clear();
+ walBuffer.close();
+ bufferClosed = true;
recoverAndCheck();
}
@@ -214,6 +220,8 @@ public class WALRecoverManagerTest {
public void testMemTableSnapshot() throws Exception {
prepareCheckpointAndWALFileForSnapshot();
WALRecoverManager.getInstance().clear();
+ walBuffer.close();
+ bufferClosed = true;
recoverAndCheck();
}
@@ -356,6 +364,8 @@ public class WALRecoverManagerTest {
@Test
public void testRecoverOldWalWithEmptyTsFile() throws Exception {
+ walBuffer.close();
+ bufferClosed = true;
// old version of wal is generated by
prepareCheckpointAndWALFileForSnapshot() in v1.3
String oldWalPathStr =
this.getClass().getClassLoader().getResource("oldwal").getFile();
File oldWalFileDir = new File(oldWalPathStr);
@@ -365,8 +375,9 @@ public class WALRecoverManagerTest {
}
@Test
- @Ignore // this scenario needs to fix later
public void testRecoverOldWalWithBrokenTsFile() throws Exception {
+ walBuffer.close();
+ bufferClosed = true;
// old version of wal is generated by
prepareCheckpointAndWALFileForSnapshot() in v1.3
String oldWalPathStr =
this.getClass().getClassLoader().getResource("oldwal").getFile();
File oldWalFileDir = new File(oldWalPathStr);
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/SealedTsFileRecoverPerformerTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/SealedTsFileRecoverPerformerTest.java
index bc28bcd3f42..c36abcc8f50 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/SealedTsFileRecoverPerformerTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/SealedTsFileRecoverPerformerTest.java
@@ -193,6 +193,7 @@ public class SealedTsFileRecoverPerformerTest {
reader.close();
// check .resource file in memory
assertEquals(1, tsFileResource.getStartTime(DEVICE1_NAME));
+
assertEquals(2, tsFileResource.getEndTime(DEVICE1_NAME));
assertEquals(3, tsFileResource.getStartTime(DEVICE2_NAME));
assertEquals(4, tsFileResource.getEndTime(DEVICE2_NAME));
