This is an automated email from the ASF dual-hosted git repository. marklau99 pushed a commit to branch fix-npe-in-cpu-metrics in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 5b2dff6b3a623a5804fdacba1b92fec815a0a596 Author: YongzaoDan <[email protected]> AuthorDate: Mon Jun 19 18:00:22 2023 +0800 [IOTDB-5997] Improve efficiency of ConfigNode PartitionInfo loadSnapshot (#10212) --- .../persistence/executor/ConfigPlanExecutor.java | 24 ++++++++++++++++++++-- .../persistence/partition/PartitionInfo.java | 10 +++++++-- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java b/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java index e33c0ec0145..a6e7710db84 100644 --- a/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java +++ b/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java @@ -480,7 +480,16 @@ public class ConfigPlanExecutor { x -> { boolean takeSnapshotResult = true; try { + long startTime = System.currentTimeMillis(); + LOGGER.info( + "[ConfigNodeSnapshot] Start to take snapshot for {} into {}", + x.getClass().getName(), + snapshotDir.getAbsolutePath()); takeSnapshotResult = x.processTakeSnapshot(snapshotDir); + LOGGER.info( + "[ConfigNodeSnapshot] Finish to take snapshot for {}, time consumption: {} ms", + x.getClass().getName(), + System.currentTimeMillis() - startTime); } catch (TException | IOException e) { LOGGER.error("Take snapshot error: {}", e.getMessage()); takeSnapshotResult = false; @@ -493,7 +502,7 @@ public class ConfigPlanExecutor { } }); if (result.get()) { - LOGGER.info("Task snapshot success, snapshotDir: {}", snapshotDir); + LOGGER.info("[ConfigNodeSnapshot] Task snapshot success, snapshotDir: {}", snapshotDir); } return result.get(); } @@ -512,14 +521,25 @@ public class ConfigPlanExecutor { .forEach( x -> { try { + long startTime = System.currentTimeMillis(); + LOGGER.info( + "[ConfigNodeSnapshot] Start to load snapshot for {} from {}", + x.getClass().getName(), + latestSnapshotRootDir.getAbsolutePath()); x.processLoadSnapshot(latestSnapshotRootDir); + LOGGER.info( + "[ConfigNodeSnapshot] Load snapshot for {} cost {} ms", + x.getClass().getName(), + System.currentTimeMillis() - startTime); } catch (TException | IOException e) { result.set(false); LOGGER.error("Load snapshot error: {}", e.getMessage()); } }); if (result.get()) { - LOGGER.info("Load snapshot success, latestSnapshotRootDir: {}", latestSnapshotRootDir); + LOGGER.info( + "[ConfigNodeSnapshot] Load snapshot success, latestSnapshotRootDir: {}", + latestSnapshotRootDir); } } diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java b/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java index 0a31ef5de68..c6a74c1313e 100644 --- a/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java +++ b/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java @@ -71,10 +71,11 @@ import org.apache.thrift.transport.TIOStreamTransport; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.BufferedInputStream; import java.io.File; -import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; +import java.nio.file.Files; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -100,6 +101,9 @@ public class PartitionInfo implements SnapshotProcessor { private static final Logger LOGGER = LoggerFactory.getLogger(PartitionInfo.class); + // Allocate 8MB buffer for load snapshot of PartitionInfo + private static final int PARTITION_TABLE_BUFFER_SIZE = 32 * 1024 * 1024; + /** For Cluster Partition */ // For allocating Regions private final AtomicInteger nextRegionGroupId; @@ -825,7 +829,9 @@ public class PartitionInfo implements SnapshotProcessor { return; } - try (FileInputStream fileInputStream = new FileInputStream(snapshotFile); + try (BufferedInputStream fileInputStream = + new BufferedInputStream( + Files.newInputStream(snapshotFile.toPath()), PARTITION_TABLE_BUFFER_SIZE); TIOStreamTransport tioStreamTransport = new TIOStreamTransport(fileInputStream)) { TProtocol protocol = new TBinaryProtocol(tioStreamTransport); // before restoring a snapshot, clear all old data
