This is an automated email from the ASF dual-hosted git repository. east pushed a commit to branch nvmlogging in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 1fa591da92691ed0093bc10b4f6d89eb703bd47a Author: mdf369 <[email protected]> AuthorDate: Mon Jan 13 15:35:09 2020 +0800 fix bugs --- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 2 +- .../iotdb/db/engine/flush/MemTableFlushTask.java | 2 ++ .../iotdb/db/nvm/rescon/NVMPrimitiveArrayPool.java | 2 +- .../writelog/recover/TsFileRecoverPerformer.java | 34 +++++++++++++--------- 4 files changed, 25 insertions(+), 15 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 250d39e..e6fe35f 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -591,7 +591,7 @@ public class IoTDBConfig { } public boolean isEnableWal() { - return enableWal; + return !enableNVM && enableWal; } public void setEnableWal(boolean enableWal) { diff --git a/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java b/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java index 8873dc4..cb5615c 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java @@ -27,6 +27,7 @@ import org.apache.iotdb.db.engine.flush.pool.FlushSubTaskPoolManager; import org.apache.iotdb.db.engine.memtable.IMemTable; import org.apache.iotdb.db.engine.memtable.IWritableMemChunk; import org.apache.iotdb.db.exception.runtime.FlushRunTimeException; +import org.apache.iotdb.db.nvm.PerfMonitor; import org.apache.iotdb.db.utils.datastructure.AbstractTVList; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.utils.Pair; @@ -93,6 +94,7 @@ public class MemTableFlushTask { logger.debug( "Storage group {} memtable {}, flushing into disk: data sort time cost {} ms.", storageGroup, memTable.getVersion(), sortTime); + PerfMonitor.add("sort", sortTime); ioTaskFuture.get(); diff --git a/server/src/main/java/org/apache/iotdb/db/nvm/rescon/NVMPrimitiveArrayPool.java b/server/src/main/java/org/apache/iotdb/db/nvm/rescon/NVMPrimitiveArrayPool.java index aca3c81..e992781 100644 --- a/server/src/main/java/org/apache/iotdb/db/nvm/rescon/NVMPrimitiveArrayPool.java +++ b/server/src/main/java/org/apache/iotdb/db/nvm/rescon/NVMPrimitiveArrayPool.java @@ -44,7 +44,7 @@ public class NVMPrimitiveArrayPool { nvmSpace = NVMSpaceManager.getInstance().allocateDataSpace(size * ARRAY_SIZE, dataType, isTime); } - PerfMonitor.add("NVM.getDataList", System.currentTimeMillis() - time); + PerfMonitor.add("NVM.getDataList" + (isTime ? "Time" : "Value"), System.currentTimeMillis() - time); return nvmSpace; } diff --git a/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java b/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java index 70d6a1a..6e56541 100644 --- a/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java +++ b/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java @@ -154,8 +154,10 @@ public class TsFileRecoverPerformer { } else { time = System.currentTimeMillis(); redoLogs(restorableTsFileIOWriter); + PerfMonitor.add("TsFileRecoverPerformer.redoLogs", System.currentTimeMillis() - time); // clean logs + time = System.currentTimeMillis(); try { MultiFileLogNodeManager.getInstance() .deleteNode( @@ -218,36 +220,42 @@ public class TsFileRecoverPerformer { this.logReplayer = new LogReplayer(logNodePrefix, insertFilePath, tsFileResource.getModFile(), versionController, tsFileResource, schema, recoverMemTable, acceptUnseq); + + long time = System.currentTimeMillis(); logReplayer.replayLogs(); - try { - if (!recoverMemTable.isEmpty()) { - // flush logs + PerfMonitor.add("LogReplayer.replayLogs", System.currentTimeMillis() - time); - MemTableFlushTask tableFlushTask = new MemTableFlushTask(recoverMemTable, schema, - restorableTsFileIOWriter, tsFileResource.getFile().getParentFile().getName()); - tableFlushTask.syncFlushMemTable(); - } - // close file - restorableTsFileIOWriter.endFile(schema); - tsFileResource.serialize(); - } catch (IOException | InterruptedException | ExecutionException e) { - throw new StorageGroupProcessorException(e); - } + flushRecoverdMemtable(recoverMemTable, restorableTsFileIOWriter); } private void reloadNVMData(RestorableTsFileIOWriter restorableTsFileIOWriter) throws StorageGroupProcessorException { NVMPrimitiveMemTable recoverMemTable = new NVMPrimitiveMemTable(storageGroupId); + + long time = System.currentTimeMillis(); NVMMemtableRecoverPerformer.getInstance().reconstructMemtable(recoverMemTable, tsFileResource); + PerfMonitor.add("NVMMemtableRecoverPerformer.reconstructMemtable", System.currentTimeMillis() - time); + + flushRecoverdMemtable(recoverMemTable, restorableTsFileIOWriter); + } + + private void flushRecoverdMemtable(IMemTable recoverMemTable, RestorableTsFileIOWriter restorableTsFileIOWriter) + throws StorageGroupProcessorException { + long time; try { if (!recoverMemTable.isEmpty()) { MemTableFlushTask tableFlushTask = new MemTableFlushTask(recoverMemTable, schema, restorableTsFileIOWriter, tsFileResource.getFile().getParentFile().getName()); + + time = System.currentTimeMillis(); tableFlushTask.syncFlushMemTable(); + PerfMonitor.add("MemTableFlushTask.syncFlushMemTable", System.currentTimeMillis() - time); } // close file + time = System.currentTimeMillis(); restorableTsFileIOWriter.endFile(schema); tsFileResource.serialize(); + PerfMonitor.add("TsFileRecoverPerformer.closeFile", System.currentTimeMillis() - time); } catch (IOException | InterruptedException | ExecutionException e) { throw new StorageGroupProcessorException(e); }
