This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch ty/TableModelGrammar in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 3af5813e64904f367b867a2df63594065948d461 Author: 133tosakarin <[email protected]> AuthorDate: Fri Jul 26 13:28:16 2024 +0800 fix md5 suffix too many (#13033) (cherry picked from commit f0363a758cc2cb9e8f6e0978f2c9922d90e65f2f) --- .../org/apache/iotdb/consensus/IStateMachine.java | 3 +-- .../org/apache/iotdb/consensus/common/Utils.java | 6 ++--- .../consensus/iot/IoTConsensusServerImpl.java | 30 ++++++++++------------ .../iotdb/consensus/ratis/SnapshotStorage.java | 10 ++++---- .../apache/iotdb/consensus/ratis/SnapshotTest.java | 5 ++-- .../dataregion/DataRegionStateMachine.java | 6 ++--- 6 files changed, 26 insertions(+), 34 deletions(-) diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/IStateMachine.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/IStateMachine.java index d1f59d7a7b7..d9db3a836b2 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/IStateMachine.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/IStateMachine.java @@ -29,7 +29,6 @@ import org.apache.iotdb.consensus.common.request.IConsensusRequest; import javax.annotation.concurrent.ThreadSafe; import java.io.File; -import java.nio.file.Path; import java.util.List; import java.util.function.Function; @@ -117,7 +116,7 @@ public interface IStateMachine { * @param latestSnapshotRootDir dir where the latest snapshot sits * @return List of real snapshot files. */ - default List<Path> getSnapshotFiles(File latestSnapshotRootDir) { + default List<File> getSnapshotFiles(File latestSnapshotRootDir) { return Utils.listAllRegularFilesRecursively(latestSnapshotRootDir); } diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/common/Utils.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/common/Utils.java index 82d7ea43a1b..106157c8f66 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/common/Utils.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/common/Utils.java @@ -37,8 +37,8 @@ public class Utils { private Utils() {} - public static List<Path> listAllRegularFilesRecursively(File rootDir) { - List<Path> allFiles = new ArrayList<>(); + public static List<File> listAllRegularFilesRecursively(File rootDir) { + List<File> allFiles = new ArrayList<>(); try { Files.walkFileTree( rootDir.toPath(), @@ -51,7 +51,7 @@ public class Utils { @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { if (attrs.isRegularFile()) { - allFiles.add(file); + allFiles.add(file.toFile()); } return FileVisitResult.CONTINUE; } diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java index d3cec61dd30..300f3725de5 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java @@ -298,23 +298,18 @@ public class IoTConsensusServerImpl { public void transmitSnapshot(Peer targetPeer) throws ConsensusGroupModifyPeerException { File snapshotDir = new File(storageDir, newSnapshotDirName); - List<Path> snapshotPaths = stateMachine.getSnapshotFiles(snapshotDir); + List<File> snapshotPaths = stateMachine.getSnapshotFiles(snapshotDir); AtomicLong snapshotSizeSumAtomic = new AtomicLong(); StringBuilder allFilesStr = new StringBuilder(); snapshotPaths.forEach( - path -> { - try { - long fileSize = Files.size(path); - snapshotSizeSumAtomic.addAndGet(fileSize); - allFilesStr - .append("\n") - .append(path) - .append(" ") - .append(humanReadableByteCountSI(fileSize)); - } catch (IOException e) { - logger.error( - "[SNAPSHOT TRANSMISSION] Calculate snapshot file's size fail: {}", path, e); - } + file -> { + long fileSize = file.length(); + snapshotSizeSumAtomic.addAndGet(fileSize); + allFilesStr + .append("\n") + .append(file.getName()) + .append(" ") + .append(humanReadableByteCountSI(fileSize)); }); final long snapshotSizeSum = snapshotSizeSumAtomic.get(); long transitedSnapshotSizeSum = 0; @@ -329,8 +324,9 @@ public class IoTConsensusServerImpl { "[SNAPSHOT TRANSMISSION] All the files below shell be transmitted: {}", allFilesStr); try (SyncIoTConsensusServiceClient client = syncClientManager.borrowClient(targetPeer.getEndpoint())) { - for (Path path : snapshotPaths) { - SnapshotFragmentReader reader = new SnapshotFragmentReader(newSnapshotDirName, path); + for (File file : snapshotPaths) { + SnapshotFragmentReader reader = + new SnapshotFragmentReader(newSnapshotDirName, file.toPath()); try { while (reader.hasNext()) { // TODO: zero copy ? @@ -356,7 +352,7 @@ public class IoTConsensusServerImpl { humanReadableByteCountSI(snapshotSizeSum), CommonDateTimeUtils.convertMillisecondToDurationStr( (System.nanoTime() - startTime) / 1_000_000), - path); + file); } finally { reader.close(); } diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/ratis/SnapshotStorage.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/ratis/SnapshotStorage.java index 52d69c74bbb..5b1d9928ced 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/ratis/SnapshotStorage.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/ratis/SnapshotStorage.java @@ -155,24 +155,24 @@ public class SnapshotStorage implements StateMachineStorage { } TermIndex snapshotTermIndex = Utils.getTermIndexFromDir(latestSnapshotDir); - List<Path> actualSnapshotFiles = applicationStateMachine.getSnapshotFiles(latestSnapshotDir); + List<File> actualSnapshotFiles = applicationStateMachine.getSnapshotFiles(latestSnapshotDir); if (actualSnapshotFiles == null) { return null; } List<FileInfo> fileInfos = new ArrayList<>(); - for (Path file : actualSnapshotFiles) { - if (file.endsWith(".md5")) { + for (File file : actualSnapshotFiles) { + if (file.getName().endsWith(".md5")) { continue; } FileInfo fileInfo; try { if (getSnapshotDir() == null) { // For regions that place the snapshot in default sm folder, use relative path - fileInfo = new FileInfo(file, null); + fileInfo = new FileInfo(file.toPath(), null); } else { // For regions that have a separate snapshot installation path, use absolute path - fileInfo = new FileInfo(file.toRealPath(), null); + fileInfo = new FileInfo(file.toPath().toRealPath(), null); } } catch (IOException e) { logger.warn("{} cannot resolve real path of {} due to ", this, file, e); diff --git a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/ratis/SnapshotTest.java b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/ratis/SnapshotTest.java index c5447075181..c43699dbcce 100644 --- a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/ratis/SnapshotTest.java +++ b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/ratis/SnapshotTest.java @@ -39,7 +39,6 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.FileWriter; import java.io.IOException; -import java.nio.file.Path; import java.util.Collections; import java.util.List; import java.util.Scanner; @@ -174,7 +173,7 @@ public class SnapshotTest { } @Override - public List<Path> getSnapshotFiles(File latestSnapshotRootDir) { + public List<File> getSnapshotFiles(File latestSnapshotRootDir) { File log = new File(latestSnapshotRootDir.getAbsolutePath() + File.separator + "record"); Assert.assertTrue(log.exists()); Scanner scanner = null; @@ -188,7 +187,7 @@ public class SnapshotTest { } Assert.assertNotNull(scanner); - return Collections.singletonList(new File(latestSnapshotRootDir, relativePath).toPath()); + return Collections.singletonList(new File(latestSnapshotRootDir, relativePath)); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachine.java index 87c91ba25d4..240e77580b2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachine.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachine.java @@ -52,10 +52,8 @@ import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; -import java.nio.file.Path; import java.util.ArrayList; import java.util.List; -import java.util.stream.Collectors; public class DataRegionStateMachine extends BaseStateMachine { @@ -241,13 +239,13 @@ public class DataRegionStateMachine extends BaseStateMachine { } @Override - public List<Path> getSnapshotFiles(File latestSnapshotRootDir) { + public List<File> getSnapshotFiles(File latestSnapshotRootDir) { try { return new SnapshotLoader( latestSnapshotRootDir.getAbsolutePath(), region.getDatabaseName(), region.getDataRegionId()) - .getSnapshotFileInfo().stream().map(File::toPath).collect(Collectors.toList()); + .getSnapshotFileInfo(); } catch (IOException e) { logger.error( "Meets error when getting snapshot files for {}-{}",
