This is an automated email from the ASF dual-hosted git repository. lta pushed a commit to branch reimpl_sync in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 2060feaa0517de2a2a3d535613c4bcabbcdf0694 Author: lta <[email protected]> AuthorDate: Mon Sep 2 17:49:52 2019 +0800 add snapshot unit test --- .../sender/recover/ISyncSenderLogAnalyzer.java | 2 +- .../sync/sender/recover/SyncSenderLogAnalyzer.java | 9 +- .../sync/sender/transfer/DataTransferManager.java | 6 +- .../java/org/apache/iotdb/db/utils/SyncUtils.java | 2 +- .../sender/recover/SyncSenderLogAnalyzerTest.java | 18 +++ .../sender/transfer/DataTransferManagerTest.java | 135 +++++++++++++++++++++ 6 files changed, 165 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/sync/sender/recover/ISyncSenderLogAnalyzer.java b/server/src/main/java/org/apache/iotdb/db/sync/sender/recover/ISyncSenderLogAnalyzer.java index 3d7a356..652839b 100644 --- a/server/src/main/java/org/apache/iotdb/db/sync/sender/recover/ISyncSenderLogAnalyzer.java +++ b/server/src/main/java/org/apache/iotdb/db/sync/sender/recover/ISyncSenderLogAnalyzer.java @@ -34,6 +34,6 @@ public interface ISyncSenderLogAnalyzer { void loadLogger(Set<String> deletedFiles, Set<String> newFiles); - void clearLogger(Set<String> currentLocalFiles); + void updateLastLocalFile(Set<String> currentLocalFiles); } diff --git a/server/src/main/java/org/apache/iotdb/db/sync/sender/recover/SyncSenderLogAnalyzer.java b/server/src/main/java/org/apache/iotdb/db/sync/sender/recover/SyncSenderLogAnalyzer.java index 6eaec50..5f97f05 100644 --- a/server/src/main/java/org/apache/iotdb/db/sync/sender/recover/SyncSenderLogAnalyzer.java +++ b/server/src/main/java/org/apache/iotdb/db/sync/sender/recover/SyncSenderLogAnalyzer.java @@ -58,7 +58,7 @@ public class SyncSenderLogAnalyzer implements ISyncSenderLogAnalyzer { loadLogger(deletedFiles, newFiles); lastLocalFiles.removeAll(deletedFiles); lastLocalFiles.addAll(newFiles); - clearLogger(lastLocalFiles); + updateLastLocalFile(lastLocalFiles); } FileUtils.deleteDirectory(new File(senderPath, Constans.DATA_SNAPSHOT_NAME)); syncLogFile.delete(); @@ -67,6 +67,7 @@ public class SyncSenderLogAnalyzer implements ISyncSenderLogAnalyzer { @Override public void loadLastLocalFiles(Set<String> lastLocalFiles) { if (!lastLocalFile.exists()) { + LOGGER.info("last local file {} doesn't exist.", syncLogFile.getAbsolutePath()); return; } try (BufferedReader br = new BufferedReader(new FileReader(lastLocalFile))) { @@ -83,6 +84,10 @@ public class SyncSenderLogAnalyzer implements ISyncSenderLogAnalyzer { @Override public void loadLogger(Set<String> deletedFiles, Set<String> newFiles) { + if (!syncLogFile.exists()) { + LOGGER.info("log file {} doesn't exist.", syncLogFile.getAbsolutePath()); + return; + } try (BufferedReader br = new BufferedReader(new FileReader(syncLogFile))) { String line; int mode = 0; @@ -107,7 +112,7 @@ public class SyncSenderLogAnalyzer implements ISyncSenderLogAnalyzer { } @Override - public void clearLogger(Set<String> currentLocalFiles) { + public void updateLastLocalFile(Set<String> currentLocalFiles) { try (BufferedWriter bw = new BufferedWriter(new FileWriter(currentLocalFile))) { for (String line : currentLocalFiles) { bw.write(line); diff --git a/server/src/main/java/org/apache/iotdb/db/sync/sender/transfer/DataTransferManager.java b/server/src/main/java/org/apache/iotdb/db/sync/sender/transfer/DataTransferManager.java index 6331d2c..aecf59e 100644 --- a/server/src/main/java/org/apache/iotdb/db/sync/sender/transfer/DataTransferManager.java +++ b/server/src/main/java/org/apache/iotdb/db/sync/sender/transfer/DataTransferManager.java @@ -524,18 +524,18 @@ public class DataTransferManager implements IDataTransferManager { * * @param file new tsfile to be synced */ - private File makeFileSnapshot(File file) throws IOException { + File makeFileSnapshot(File file) throws IOException { File snapshotFile = SyncUtils.getSnapshotFile(file); if (!snapshotFile.getParentFile().exists()) { snapshotFile.getParentFile().mkdirs(); } Path link = FileSystems.getDefault().getPath(snapshotFile.getAbsolutePath()); - Path target = FileSystems.getDefault().getPath(snapshotFile.getAbsolutePath()); + Path target = FileSystems.getDefault().getPath(file.getAbsolutePath()); Files.createLink(link, target); link = FileSystems.getDefault() .getPath(snapshotFile.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX); target = FileSystems.getDefault() - .getPath(snapshotFile.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX); + .getPath(file.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX); Files.createLink(link, target); return snapshotFile; } diff --git a/server/src/main/java/org/apache/iotdb/db/utils/SyncUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/SyncUtils.java index 13247cb..599edac 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/SyncUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/SyncUtils.java @@ -38,7 +38,7 @@ public class SyncUtils { * sender. */ public static File getSnapshotFile(File file) { - String relativeFilePath = file.getParent() + File.separator + file.getName(); + String relativeFilePath = file.getParentFile().getName() + File.separator + file.getName(); String snapshotDir = SyncSenderDescriptor.getInstance().getConfig().getSnapshotPath(); if (!new File(snapshotDir).exists()) { new File(snapshotDir).mkdirs(); diff --git a/server/src/test/java/org/apache/iotdb/db/sync/sender/recover/SyncSenderLogAnalyzerTest.java b/server/src/test/java/org/apache/iotdb/db/sync/sender/recover/SyncSenderLogAnalyzerTest.java index c30eb8f..71c9629 100644 --- a/server/src/test/java/org/apache/iotdb/db/sync/sender/recover/SyncSenderLogAnalyzerTest.java +++ b/server/src/test/java/org/apache/iotdb/db/sync/sender/recover/SyncSenderLogAnalyzerTest.java @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ package org.apache.iotdb.db.sync.sender.recover; import java.io.File; diff --git a/server/src/test/java/org/apache/iotdb/db/sync/sender/transfer/DataTransferManagerTest.java b/server/src/test/java/org/apache/iotdb/db/sync/sender/transfer/DataTransferManagerTest.java new file mode 100644 index 0000000..ef49dd5 --- /dev/null +++ b/server/src/test/java/org/apache/iotdb/db/sync/sender/transfer/DataTransferManagerTest.java @@ -0,0 +1,135 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.sync.sender.transfer; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Random; +import java.util.Set; +import org.apache.iotdb.db.conf.IoTDBConstant; +import org.apache.iotdb.db.conf.directories.DirectoryManager; +import org.apache.iotdb.db.engine.storagegroup.TsFileResource; +import org.apache.iotdb.db.exception.DiskSpaceInsufficientException; +import org.apache.iotdb.db.exception.StartupException; +import org.apache.iotdb.db.exception.StorageEngineException; +import org.apache.iotdb.db.sync.sender.conf.SyncSenderConfig; +import org.apache.iotdb.db.sync.sender.conf.SyncSenderDescriptor; +import org.apache.iotdb.db.sync.sender.recover.SyncSenderLogAnalyzer; +import org.apache.iotdb.db.utils.EnvironmentUtils; +import org.apache.iotdb.db.utils.FilePathUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DataTransferManagerTest { + + private static final Logger logger = LoggerFactory.getLogger(DataTransferManagerTest.class); + private DataTransferManager manager = DataTransferManager.getInstance(); + private SyncSenderConfig config = SyncSenderDescriptor.getInstance().getConfig(); + private String dataDir; + private SyncSenderLogAnalyzer senderLogAnalyzer; + + @Before + public void setUp() + throws IOException, InterruptedException, StartupException, DiskSpaceInsufficientException { + EnvironmentUtils.envSetUp(); + dataDir = new File(DirectoryManager.getInstance().getNextFolderForSequenceFile()) + .getParentFile().getAbsolutePath(); + config.update(dataDir); + senderLogAnalyzer = new SyncSenderLogAnalyzer(config.getSenderFolderPath()); + } + + @After + public void tearDown() throws InterruptedException, IOException, StorageEngineException { + EnvironmentUtils.cleanEnv(); + } + + @Test + public void makeFileSnapshot() throws IOException { + Map<String, Set<File>> allFileList = new HashMap<>(); + + Random r = new Random(0); + for (int i = 0; i < 3; i++) { + for (int j = 0; j < 5; j++) { + if (!allFileList.containsKey(String.valueOf(i))) { + allFileList.put(String.valueOf(i), new HashSet<>()); + } + String rand = String.valueOf(r.nextInt(10000)); + String fileName = FilePathUtils.regularizePath(dataDir) + IoTDBConstant.SEQUENCE_FLODER_NAME + + File.separator + i + + File.separator + rand; + File file = new File(fileName); + allFileList.get(String.valueOf(i)).add(file); + if (!file.getParentFile().exists()) { + file.getParentFile().mkdirs(); + } + if (!file.exists() && !file.createNewFile()) { + logger.error("Can not create new file {}", file.getPath()); + } + if (!new File(file.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX).exists() + && !new File(file.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX).createNewFile()) { + logger.error("Can not create new file {}", file.getPath()); + } + } + } + + Map<String, Set<String>> dataFileMap = new HashMap<>(); + File sequenceFile = new File(dataDir, IoTDBConstant.SEQUENCE_FLODER_NAME); + for(File sgFile: sequenceFile.listFiles()){ + dataFileMap.putIfAbsent(sgFile.getName(), new HashSet<>()); + for (File tsfile : sgFile.listFiles()) { + if (!tsfile.getName().endsWith(TsFileResource.RESOURCE_SUFFIX)) { + manager.makeFileSnapshot(tsfile); + } + dataFileMap.get(sgFile.getName()).add(tsfile.getName()); + } + } + + assert new File(config.getSenderFolderPath()).exists(); + assert new File(config.getSnapshotPath()).exists(); + + Map<String, Set<String>> snapFileMap = new HashMap<>(); + for(File sgFile: new File(config.getSnapshotPath()).listFiles()){ + snapFileMap.putIfAbsent(sgFile.getName(), new HashSet<>()); + for(File snapshotTsfile: sgFile.listFiles()){ + snapFileMap.get(sgFile.getName()).add(snapshotTsfile.getName()); + } + } + + assert dataFileMap.size() == snapFileMap.size(); + for(Entry<String, Set<String>> entry: dataFileMap.entrySet()){ + String sg = entry.getKey(); + Set<String> tsfiles = entry.getValue(); + assert snapFileMap.containsKey(sg); + assert snapFileMap.get(sg).size() == tsfiles.size(); + assert snapFileMap.get(sg).containsAll(tsfiles); + } + + assert !new File(config.getLastFileInfo()).exists(); + senderLogAnalyzer.recover(); + assert !new File(config.getSnapshotPath()).exists(); + assert new File(config.getLastFileInfo()).exists(); + } +} \ No newline at end of file
