This is an automated email from the ASF dual-hosted git repository.
tanxinyu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 1c83bc0d5a3 Do not set read only in compaction module (#12311)
1c83bc0d5a3 is described below
commit 1c83bc0d5a3c017538958f14309608b69badf034
Author: shuwenwei <[email protected]>
AuthorDate: Thu Apr 11 10:52:09 2024 +0800
Do not set read only in compaction module (#12311)
---
.../execute/exception/CompactionExceptionHandler.java | 15 +++++----------
.../execute/recover/CompactionRecoverTask.java | 18 ++++--------------
.../execute/task/AbstractCompactionTask.java | 10 +++++-----
.../execute/task/InnerSpaceCompactionTask.java | 2 +-
.../impl/RewriteCrossSpaceCompactionSelector.java | 2 +-
.../inner/InnerSpaceCompactionExceptionTest.java | 5 +----
6 files changed, 17 insertions(+), 35 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/exception/CompactionExceptionHandler.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/exception/CompactionExceptionHandler.java
index 12039324339..5c663e37955 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/exception/CompactionExceptionHandler.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/exception/CompactionExceptionHandler.java
@@ -102,23 +102,19 @@ public class CompactionExceptionHandler {
if (!handleSuccess) {
LOGGER.error(
"[Compaction][ExceptionHandler] Fail to handle {} space compaction
exception, "
- + "set allowCompaction to false in {}",
+ + "storage group is {}",
compactionType,
fullStorageGroupName);
- tsFileManager.setAllowCompaction(false);
} else {
FileUtils.delete(logFile);
}
} catch (IOException e) {
// catch throwable when handling exception
- // set the allowCompaction to false
LOGGER.error(
"[Compaction][ExceptionHandler] exception occurs when handling
exception in {} space compaction. "
- + "Set allowCompaction to false in {}",
+ + "storage group is {}",
compactionType,
- fullStorageGroupName,
- e);
- tsFileManager.setAllowCompaction(false);
+ fullStorageGroupName);
}
}
@@ -209,7 +205,7 @@ public class CompactionExceptionHandler {
/**
* Some source files are lost, check if all target files are complete. If
all target files are
* complete, delete the remaining source files and compaction mods files. If
some target files are
- * not complete, set the allowCompaction in tsFileManager to false and print
some error logs.
+ * not complete, print some error logs.
*
* @throws IOException if the io operations on file fails
*/
@@ -252,8 +248,7 @@ public class CompactionExceptionHandler {
if (!TsFileUtils.isTsFileComplete(targetResource.getTsFile())) {
LOGGER.error(
"{} [Compaction][ExceptionHandler] target file {} is not complete,
"
- + "and some source files {} is lost, do nothing. "
- + "Set allowCompaction to false",
+ + "and some source files {} is lost, do nothing.",
fullStorageGroupName,
targetResource,
lostSourceResources);
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/recover/CompactionRecoverTask.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/recover/CompactionRecoverTask.java
index 7f238ba02f7..a504df6e7cb 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/recover/CompactionRecoverTask.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/recover/CompactionRecoverTask.java
@@ -19,8 +19,6 @@
package
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.recover;
-import org.apache.iotdb.commons.cluster.NodeStatus;
-import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils;
@@ -113,10 +111,7 @@ public class CompactionRecoverTask {
LOGGER.error("Recover compaction error", e);
} finally {
if (!recoverSuccess) {
- LOGGER.error(
- "{} [Compaction][Recover] Failed to recover compaction, set
allowCompaction to false",
- fullStorageGroupName);
- tsFileManager.setAllowCompaction(false);
+ LOGGER.error("{} [Compaction][Recover] Failed to recover compaction",
fullStorageGroupName);
} else {
if (compactionLogFile.exists()) {
try {
@@ -127,12 +122,10 @@ public class CompactionRecoverTask {
FileUtils.delete(compactionLogFile);
} catch (IOException e) {
LOGGER.error(
- "{} [Compaction][Recover] Exception occurs while deleting log
file {}, "
- + "set allowCompaction to false",
+ "{} [Compaction][Recover] Exception occurs while deleting log
file {}",
fullStorageGroupName,
compactionLogFile,
e);
- tsFileManager.setAllowCompaction(false);
}
}
}
@@ -190,8 +183,7 @@ public class CompactionRecoverTask {
CompactionUtils.deleteCompactionModsFile(sourceTsFileResourceList,
Collections.emptyList());
} catch (IOException e) {
LOGGER.error(
- "{} [Compaction][Recover] Exception occurs while deleting compaction
mods file, "
- + "set allowCompaction to false",
+ "{} [Compaction][Recover] Exception occurs while deleting compaction
mods file",
fullStorageGroupName,
e);
return false;
@@ -265,11 +257,9 @@ public class CompactionRecoverTask {
|| !TsFileUtils.isTsFileComplete(new
TsFileResource(targetFile).getTsFile())) {
LOGGER.error(
"{} [Compaction][ExceptionHandler] target file {} is not complete,
"
- + "and some source files is lost, do nothing. "
- + "Set allowCompaction to false",
+ + "and some source files is lost, do nothing.",
fullStorageGroupName,
targetFileIdentifier.getFilePath());
-
CommonDescriptor.getInstance().getConfig().setNodeStatus(NodeStatus.ReadOnly);
return false;
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/AbstractCompactionTask.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/AbstractCompactionTask.java
index 76053e6e936..a79952d49ca 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/AbstractCompactionTask.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/AbstractCompactionTask.java
@@ -19,8 +19,6 @@
package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task;
-import org.apache.iotdb.commons.cluster.NodeStatus;
-import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.service.metrics.CompactionMetrics;
@@ -322,9 +320,11 @@ public abstract class AbstractCompactionTask {
dataRegionId,
this,
e);
- tsFileManager.setAllowCompaction(false);
- LOGGER.error("stop compaction because of exception during recovering");
-
CommonDescriptor.getInstance().getConfig().setNodeStatus(NodeStatus.ReadOnly);
+ // Do not set allow compaction to false here. To keep the error
environment, mark all source
+ // files in memory to avoid compaction.
+ for (TsFileResource sourceTsFileResource : getAllSourceTsFiles()) {
+
sourceTsFileResource.setTsFileRepairStatus(TsFileRepairStatus.CAN_NOT_REPAIR);
+ }
}
protected void insertFilesToTsFileManager(List<TsFileResource> tsFiles)
throws IOException {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java
index fea3075bde2..01399c8d5fe 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java
@@ -169,7 +169,7 @@ public class InnerSpaceCompactionTask extends
AbstractCompactionTask {
recoverMemoryStatus = true;
LOGGER.info(
"{}-{} [Compaction] {} InnerSpaceCompaction task starts with {} files,
"
- + "total file size is {} MB, memory cost is {} MB",
+ + "total file size is {} MB, estimated memory cost is {} MB",
storageGroupName,
dataRegionId,
sequence ? "Sequence" : "Unsequence",
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java
index f782d186b10..43daf7b0e13 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java
@@ -363,7 +363,7 @@ public class RewriteCrossSpaceCompactionSelector implements
ICrossSpaceSelector
"{} [{}] Total source files: {} seqFiles, {} unseqFiles. "
+ "Candidate source files: {} seqFiles, {} unseqFiles. "
+ "Selected source files: {} seqFiles, "
- + "{} unseqFiles, total memory cost {} MB, "
+ + "{} unseqFiles, estimated memory cost {} MB, "
+ "total selected file size is {} MB, "
+ "total selected seq file size is {} MB, "
+ "total selected unseq file size is {} MB, "
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSpaceCompactionExceptionTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSpaceCompactionExceptionTest.java
index aa54be5766a..1087c56863b 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSpaceCompactionExceptionTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSpaceCompactionExceptionTest.java
@@ -215,8 +215,7 @@ public class InnerSpaceCompactionExceptionTest extends
AbstractInnerSpaceCompact
}
/**
- * Test some source files are lost and target file is not complete. System
should be set to read
- * only at this time.
+ * Test some source files are lost and target file is not complete.
*
* @throws Exception
*/
@@ -265,8 +264,6 @@ public class InnerSpaceCompactionExceptionTest extends
AbstractInnerSpaceCompact
Assert.assertTrue(resource.resourceFileExists());
Assert.assertTrue(resource.getTsFile().exists());
}
-
- Assert.assertFalse(tsFileManager.isAllowCompaction());
}
/**
