This is an automated email from the ASF dual-hosted git repository. sunzesong pushed a commit to branch jira_782 in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 463321f87b9098adc6e50176374943d0ccfd3525 Author: samperson1997 <[email protected]> AuthorDate: Sat Jun 27 02:27:29 2020 +0800 [Experimental] Clear mlog when creating snapshot --- .../org/apache/iotdb/db/metadata/MLogWriter.java | 13 +++++++-- .../org/apache/iotdb/db/metadata/MManager.java | 34 ++++++++++------------ .../java/org/apache/iotdb/db/metadata/MTree.java | 5 ++++ 3 files changed, 31 insertions(+), 21 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MLogWriter.java b/server/src/main/java/org/apache/iotdb/db/metadata/MLogWriter.java index 18115d3..6b41a44 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/MLogWriter.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/MLogWriter.java @@ -24,6 +24,7 @@ import java.io.File; import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; +import java.nio.file.Files; import java.util.Map; import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory; import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan; @@ -34,6 +35,7 @@ import org.slf4j.LoggerFactory; public class MLogWriter { private static final Logger logger = LoggerFactory.getLogger(MLogWriter.class); + private File logFile; private BufferedWriter writer; private int lineNumber; @@ -47,7 +49,7 @@ public class MLogWriter { } } - File logFile = SystemFileFactory.INSTANCE.getFile(schemaDir + File.separator + logFileName); + logFile = SystemFileFactory.INSTANCE.getFile(schemaDir + File.separator + logFileName); FileWriter fileWriter = new FileWriter(logFile, true); writer = new BufferedWriter(fileWriter); } @@ -160,6 +162,14 @@ public class MLogWriter { FSFactoryProducer.getFSFactory().moveFile(tmpLogFile, logFile); } + public void clear() throws IOException { + writer.close(); + Files.delete(logFile.toPath()); + FileWriter fileWriter = new FileWriter(logFile, true); + writer = new BufferedWriter(fileWriter); + lineNumber = 0; + } + private void newLine() throws IOException { writer.newLine(); writer.flush(); @@ -172,7 +182,6 @@ public class MLogWriter { /** * only used for initialize a mlog file writer. - * @param number */ void setLineNumber(int number) { lineNumber = number; diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java index d247342..19b4118 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java @@ -228,10 +228,7 @@ public class MManager { } /** - * - * @param logFile * @return line number of the logFile - * @throws IOException */ @SuppressWarnings("squid:S3776") private int initFromLog(File logFile) throws IOException { @@ -248,8 +245,10 @@ public class MManager { } else { mtree = MTree.deserializeFrom(mtreeSnapshot); lastSnapshotLogLineNumber = mtree.getSnapshotLineNumber(); + logger + .debug("spend {} ms to deserialize mtree from snapshot", System.currentTimeMillis() - time); + } - logger.debug("spend {} ms to deserialize mtree from snapshot", System.currentTimeMillis() - time); time = System.currentTimeMillis(); // init the metadata from the operation log @@ -258,15 +257,6 @@ public class MManager { try (FileReader fr = new FileReader(logFile); BufferedReader br = new BufferedReader(fr)) { String cmd; - while (idx < mtree.getSnapshotLineNumber()) { - cmd = br.readLine(); - if (cmd == null) { - throw new IOException(String - .format("mtree snapshot file has %d lines but the mlog.txt has only %d lines.", - mtree.getSnapshotLineNumber(), idx)); - } - idx++; - } while ((cmd = br.readLine()) != null) { try { operation(cmd); @@ -276,7 +266,8 @@ public class MManager { } } } - logger.debug("spend {} ms to deserialize mtree from mlog.txt", System.currentTimeMillis() - time); + logger.debug("spend {} ms to deserialize mtree from mlog.txt", + System.currentTimeMillis() - time); return idx; } else if (mtreeSnapshot.exists()) { throw new IOException("mtree snapshot file exists but mlog.txt does not exist."); @@ -1804,15 +1795,17 @@ public class MManager { } if (System.currentTimeMillis() - logFile.lastModified() >= mtreeSnapshotThresholdTime && logWriter.getLineNumber() > lastSnapshotLogLineNumber) { - logger.info("Start creating MTree snapshot, because {} ms elaspse.", System.currentTimeMillis() - logFile.lastModified()); + logger.info("Start creating MTree snapshot, because {} ms elapse.", + System.currentTimeMillis() - logFile.lastModified()); createMTreeSnapshot(); } else if (logWriter.getLineNumber() - lastSnapshotLogLineNumber >= mtreeSnapshotInterval) { - logger.info("Start creating MTree snapshot, because of {} new lines are added.", logWriter.getLineNumber() - lastSnapshotLogLineNumber); + logger.info("Start creating MTree snapshot, because of {} new lines are added.", + logWriter.getLineNumber() - lastSnapshotLogLineNumber); createMTreeSnapshot(); } else { if (logger.isDebugEnabled()) { logger.debug( - "MTree snapshot need not be created. Current mlog line number: {}, last snapshot line number: {}, time difference from last modification: {}ms", + "MTree snapshot need not be created. Current mlog line number: {}, last snapshot line number: {}, time difference from last modification: {} ms", logWriter.getLineNumber(), lastSnapshotLogLineNumber, System.currentTimeMillis() - logFile.lastModified()); } @@ -1823,7 +1816,8 @@ public class MManager { lock.readLock().lock(); long time = System.currentTimeMillis(); try { - mtree.serializeTo(mtreeSnapshotTmpPath, logWriter.getLineNumber()); + mtree.serializeTo(mtreeSnapshotTmpPath, + logWriter.getLineNumber() + mtree.getSnapshotLineNumber()); lastSnapshotLogLineNumber = logWriter.getLineNumber(); File tmpFile = SystemFileFactory.INSTANCE.getFile(mtreeSnapshotTmpPath); File snapshotFile = SystemFileFactory.INSTANCE.getFile(mtreeSnapshotPath); @@ -1831,8 +1825,10 @@ public class MManager { Files.delete(snapshotFile.toPath()); } if (tmpFile.renameTo(snapshotFile)) { - logger.info("Finish creating MTree snapshot to {}, spend {}ms.", mtreeSnapshotPath, System.currentTimeMillis() - time); + logger.info("Finish creating MTree snapshot to {}, spend {} ms.", mtreeSnapshotPath, + System.currentTimeMillis() - time); } + logWriter.clear(); } catch (IOException e) { logger.warn("Failed to create MTree snapshot to {}", mtreeSnapshotPath, e); if (SystemFileFactory.INSTANCE.getFile(mtreeSnapshotTmpPath).exists()) { diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java b/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java index b4eb3e0..6679c34 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java @@ -922,6 +922,10 @@ public class MTree implements Serializable { } } + public void setSnapshotLineNumber(int snapshotLineNumber) { + this.snapshotLineNumber = snapshotLineNumber; + } + public int getSnapshotLineNumber() { return snapshotLineNumber; } @@ -932,6 +936,7 @@ public class MTree implements Serializable { bw.write(String.valueOf(lineNumber)); bw.newLine(); root.serializeTo(bw); + setSnapshotLineNumber(lineNumber); } }
