This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch rel/1.2
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/1.2 by this push:
new 4c027908ef7 [IOTDB-5997] Improve efficiency of ConfigNode
PartitionInfo loadSnapshot (#10212)
4c027908ef7 is described below
commit 4c027908ef70e9b4d204bf4a29109a4621221c35
Author: YongzaoDan <[email protected]>
AuthorDate: Mon Jun 19 18:00:22 2023 +0800
[IOTDB-5997] Improve efficiency of ConfigNode PartitionInfo loadSnapshot
(#10212)
---
.../persistence/executor/ConfigPlanExecutor.java | 24 ++++++++++++++++++++--
.../persistence/partition/PartitionInfo.java | 10 +++++++--
2 files changed, 30 insertions(+), 4 deletions(-)
diff --git
a/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java
b/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java
index e33c0ec0145..a6e7710db84 100644
---
a/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java
+++
b/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java
@@ -480,7 +480,16 @@ public class ConfigPlanExecutor {
x -> {
boolean takeSnapshotResult = true;
try {
+ long startTime = System.currentTimeMillis();
+ LOGGER.info(
+ "[ConfigNodeSnapshot] Start to take snapshot for {} into {}",
+ x.getClass().getName(),
+ snapshotDir.getAbsolutePath());
takeSnapshotResult = x.processTakeSnapshot(snapshotDir);
+ LOGGER.info(
+ "[ConfigNodeSnapshot] Finish to take snapshot for {}, time
consumption: {} ms",
+ x.getClass().getName(),
+ System.currentTimeMillis() - startTime);
} catch (TException | IOException e) {
LOGGER.error("Take snapshot error: {}", e.getMessage());
takeSnapshotResult = false;
@@ -493,7 +502,7 @@ public class ConfigPlanExecutor {
}
});
if (result.get()) {
- LOGGER.info("Task snapshot success, snapshotDir: {}", snapshotDir);
+ LOGGER.info("[ConfigNodeSnapshot] Task snapshot success, snapshotDir:
{}", snapshotDir);
}
return result.get();
}
@@ -512,14 +521,25 @@ public class ConfigPlanExecutor {
.forEach(
x -> {
try {
+ long startTime = System.currentTimeMillis();
+ LOGGER.info(
+ "[ConfigNodeSnapshot] Start to load snapshot for {} from
{}",
+ x.getClass().getName(),
+ latestSnapshotRootDir.getAbsolutePath());
x.processLoadSnapshot(latestSnapshotRootDir);
+ LOGGER.info(
+ "[ConfigNodeSnapshot] Load snapshot for {} cost {} ms",
+ x.getClass().getName(),
+ System.currentTimeMillis() - startTime);
} catch (TException | IOException e) {
result.set(false);
LOGGER.error("Load snapshot error: {}", e.getMessage());
}
});
if (result.get()) {
- LOGGER.info("Load snapshot success, latestSnapshotRootDir: {}",
latestSnapshotRootDir);
+ LOGGER.info(
+ "[ConfigNodeSnapshot] Load snapshot success, latestSnapshotRootDir:
{}",
+ latestSnapshotRootDir);
}
}
diff --git
a/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java
b/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java
index 0a31ef5de68..c6a74c1313e 100644
---
a/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java
+++
b/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java
@@ -71,10 +71,11 @@ import org.apache.thrift.transport.TIOStreamTransport;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.BufferedInputStream;
import java.io.File;
-import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
@@ -100,6 +101,9 @@ public class PartitionInfo implements SnapshotProcessor {
private static final Logger LOGGER =
LoggerFactory.getLogger(PartitionInfo.class);
+ // Allocate 8MB buffer for load snapshot of PartitionInfo
+ private static final int PARTITION_TABLE_BUFFER_SIZE = 32 * 1024 * 1024;
+
/** For Cluster Partition */
// For allocating Regions
private final AtomicInteger nextRegionGroupId;
@@ -825,7 +829,9 @@ public class PartitionInfo implements SnapshotProcessor {
return;
}
- try (FileInputStream fileInputStream = new FileInputStream(snapshotFile);
+ try (BufferedInputStream fileInputStream =
+ new BufferedInputStream(
+ Files.newInputStream(snapshotFile.toPath()),
PARTITION_TABLE_BUFFER_SIZE);
TIOStreamTransport tioStreamTransport = new
TIOStreamTransport(fileInputStream)) {
TProtocol protocol = new TBinaryProtocol(tioStreamTransport);
// before restoring a snapshot, clear all old data
