This is an automated email from the ASF dual-hosted git repository. east pushed a commit to branch nvmlogging in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 35bf4b5d67b3a0b70dc64694a3f65924dc5af38a Author: mdf369 <[email protected]> AuthorDate: Wed Apr 15 10:38:15 2020 +0800 add perf monitor --- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 3 ++ .../java/org/apache/iotdb/db/nvm/PerfMonitor.java | 2 +- .../nvm/recover/NVMMemtableRecoverPerformer.java | 3 ++ .../iotdb/db/nvm/rescon/NVMPrimitiveArrayPool.java | 3 +- .../apache/iotdb/db/rescon/PrimitiveArrayPool.java | 3 +- .../iotdb/db/utils/datastructure/FloatTVList.java | 5 +-- .../db/utils/datastructure/NVMBinaryTVList.java | 44 ++++++++++++++++++---- .../db/utils/datastructure/NVMFloatTVList.java | 3 ++ .../iotdb/db/utils/datastructure/NVMTVList.java | 7 +--- .../iotdb/db/utils/datastructure/TVList.java | 3 ++ .../iotdb/db/writelog/io/MultiFileLogReader.java | 10 ++++- .../iotdb/db/writelog/recover/LogReplayer.java | 3 ++ .../writelog/recover/TsFileRecoverPerformer.java | 2 +- 13 files changed, 70 insertions(+), 21 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 7069c2f..d677c35 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -355,6 +355,9 @@ public class IoTDBDescriptor { Boolean.toString(conf.isEnableNVM())))); conf.setNvmDir(properties.getProperty("nvm_dir", conf.getNvmDir())); + + conf.setArraySize(Integer.parseInt(properties.getProperty("array_size", + Integer.toString(conf.getArraySize())))); } private void loadAutoCreateSchemaProps(Properties properties){ diff --git a/server/src/main/java/org/apache/iotdb/db/nvm/PerfMonitor.java b/server/src/main/java/org/apache/iotdb/db/nvm/PerfMonitor.java index 6784e31..ec9aa06 100644 --- a/server/src/main/java/org/apache/iotdb/db/nvm/PerfMonitor.java +++ b/server/src/main/java/org/apache/iotdb/db/nvm/PerfMonitor.java @@ -39,7 +39,7 @@ public class PerfMonitor { @Override public String toString() { - return name + ":\t\t\t" + timeLen + "\t\t\t" + count; + return name + "\t" + timeLen + "\t" + count; } } } diff --git a/server/src/main/java/org/apache/iotdb/db/nvm/recover/NVMMemtableRecoverPerformer.java b/server/src/main/java/org/apache/iotdb/db/nvm/recover/NVMMemtableRecoverPerformer.java index bbb3cd5..7d54bc7 100644 --- a/server/src/main/java/org/apache/iotdb/db/nvm/recover/NVMMemtableRecoverPerformer.java +++ b/server/src/main/java/org/apache/iotdb/db/nvm/recover/NVMMemtableRecoverPerformer.java @@ -8,6 +8,7 @@ import java.util.Map; import java.util.Map.Entry; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.exception.StartupException; +import org.apache.iotdb.db.nvm.PerfMonitor; import org.apache.iotdb.db.nvm.memtable.NVMPrimitiveMemTable; import org.apache.iotdb.db.nvm.space.NVMDataSpace; import org.apache.iotdb.db.nvm.space.NVMSpaceManager; @@ -30,7 +31,9 @@ public class NVMMemtableRecoverPerformer { public void init() throws StartupException { try { + long time = System.currentTimeMillis(); dataMap = recoverDataInNVM(); + PerfMonitor.add("NVMMemtableRecoverPerformer.init", System.currentTimeMillis() - time); } catch (IOException e) { throw new StartupException(e); } diff --git a/server/src/main/java/org/apache/iotdb/db/nvm/rescon/NVMPrimitiveArrayPool.java b/server/src/main/java/org/apache/iotdb/db/nvm/rescon/NVMPrimitiveArrayPool.java index 019bb27..3f1c264 100644 --- a/server/src/main/java/org/apache/iotdb/db/nvm/rescon/NVMPrimitiveArrayPool.java +++ b/server/src/main/java/org/apache/iotdb/db/nvm/rescon/NVMPrimitiveArrayPool.java @@ -2,6 +2,7 @@ package org.apache.iotdb.db.nvm.rescon; import java.util.ArrayDeque; import java.util.EnumMap; +import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.nvm.PerfMonitor; import org.apache.iotdb.db.nvm.exception.NVMSpaceManagerException; import org.apache.iotdb.db.nvm.space.NVMDataSpace; @@ -15,7 +16,7 @@ public class NVMPrimitiveArrayPool { */ private static final EnumMap<TSDataType, ArrayDeque<NVMDataSpace>> primitiveArraysMap = new EnumMap<>(TSDataType.class); - public static final int ARRAY_SIZE = 128; + public static final int ARRAY_SIZE = IoTDBDescriptor.getInstance().getConfig().getArraySize(); static { primitiveArraysMap.put(TSDataType.BOOLEAN, new ArrayDeque()); diff --git a/server/src/main/java/org/apache/iotdb/db/rescon/PrimitiveArrayPool.java b/server/src/main/java/org/apache/iotdb/db/rescon/PrimitiveArrayPool.java index dc8b72e..be9b697 100644 --- a/server/src/main/java/org/apache/iotdb/db/rescon/PrimitiveArrayPool.java +++ b/server/src/main/java/org/apache/iotdb/db/rescon/PrimitiveArrayPool.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.rescon; import java.util.ArrayDeque; import java.util.EnumMap; +import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.nvm.PerfMonitor; import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; @@ -35,7 +36,7 @@ public class PrimitiveArrayPool { */ private static final EnumMap<TSDataType, ArrayDeque> primitiveArraysMap = new EnumMap<>(TSDataType.class); - public static final int ARRAY_SIZE = 128; + public static final int ARRAY_SIZE = IoTDBDescriptor.getInstance().getConfig().getArraySize(); static { primitiveArraysMap.put(TSDataType.BOOLEAN, new ArrayDeque()); diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/FloatTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/FloatTVList.java index 578fc13..73fac89 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/FloatTVList.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/FloatTVList.java @@ -100,18 +100,15 @@ public class FloatTVList extends TVList { sortedValues = (float[][]) PrimitiveArrayPool .getInstance().getDataListsByType(TSDataType.FLOAT, size); } - System.out.println("init arr:" + (System.currentTimeMillis() - time)); PerfMonitor.add("sort-initarr", System.currentTimeMillis() - time); time = System.currentTimeMillis(); sort(0, size); - System.out.println("sort:" + (System.currentTimeMillis() - time)); PerfMonitor.add("sort-sort", System.currentTimeMillis() - time); time = System.currentTimeMillis(); clearSortedValue(); clearSortedTime(); - System.out.println("clear arr:" + (System.currentTimeMillis() - time)); PerfMonitor.add("sort-cleararr", System.currentTimeMillis() - time); sorted = true; @@ -200,6 +197,7 @@ public class FloatTVList extends TVList { updateMinTimeAndSorted(time); + long startTime = System.currentTimeMillis(); while (idx < length) { int inputRemaining = length - idx; int arrayIdx = size / ARRAY_SIZE; @@ -221,6 +219,7 @@ public class FloatTVList extends TVList { checkExpansion(); } } + PerfMonitor.add("TVList.puts", System.currentTimeMillis() - startTime); } void addBatchValue(float[] batch) { diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMBinaryTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMBinaryTVList.java index a5b270e..0b8ce51 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMBinaryTVList.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMBinaryTVList.java @@ -2,11 +2,16 @@ package org.apache.iotdb.db.utils.datastructure; import static org.apache.iotdb.db.nvm.rescon.NVMPrimitiveArrayPool.ARRAY_SIZE; +import org.apache.iotdb.db.exception.StartupException; +import org.apache.iotdb.db.nvm.rescon.NVMPrimitiveArrayPool; import org.apache.iotdb.db.nvm.space.NVMDataSpace; +import org.apache.iotdb.db.nvm.space.NVMSpaceManager; +import org.apache.iotdb.db.nvm.space.NVMSpaceMetadataManager; import org.apache.iotdb.db.rescon.PrimitiveArrayPool; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.utils.Binary; +// TODO how to organize data public class NVMBinaryTVList extends NVMTVList { // TODO @@ -21,8 +26,8 @@ public class NVMBinaryTVList extends NVMTVList { @Override public void putBinary(long timestamp, Binary value) { checkExpansion(); - int arrayIndex = size / ARRAY_SIZE; - int elementIndex = size % ARRAY_SIZE; + int arrayIndex = size; + int elementIndex = 0; minTime = minTime <= timestamp ? minTime : timestamp; timestamps.get(arrayIndex).setData(elementIndex, timestamp); values.get(arrayIndex).setData(elementIndex, value); @@ -33,12 +38,22 @@ public class NVMBinaryTVList extends NVMTVList { } @Override + protected void checkExpansion() { + NVMDataSpace valueSpace = expandValues(); + NVMDataSpace timeSpace = NVMPrimitiveArrayPool + .getInstance().getPrimitiveDataListByType(TSDataType.INT64, true); + timestamps.add(timeSpace); + NVMSpaceMetadataManager + .getInstance().registerTVSpace(timeSpace, valueSpace, sgId, deviceId, measurementId); + } + + @Override public Binary getBinary(int index) { if (index >= size) { throw new ArrayIndexOutOfBoundsException(index); } - int arrayIndex = index / ARRAY_SIZE; - int elementIndex = index % ARRAY_SIZE; + int arrayIndex = index; + int elementIndex = 0; return (Binary) values.get(arrayIndex).getData(elementIndex); } @@ -73,8 +88,8 @@ public class NVMBinaryTVList extends NVMTVList { int arrayIndex = 0; int elementIndex = 0; for (int i = 0; i < size; i++) { - long time = (long) timestamps.get(arrayIndex).getData(elementIndex); - Binary value = (Binary) values.get(arrayIndex).getData(elementIndex); + long time = (long) timestamps.get(i).getData(0); + Binary value = (Binary) values.get(i).getData(0); tempTimestampsForSort[arrayIndex][elementIndex] = time; tempValuesForSort[arrayIndex][elementIndex] = value; @@ -94,8 +109,8 @@ public class NVMBinaryTVList extends NVMTVList { long time = tempTimestampsForSort[arrayIndex][elementIndex]; Binary value = tempValuesForSort[arrayIndex][elementIndex]; - timestamps.get(arrayIndex).setData(elementIndex, time); - values.get(arrayIndex).setData(elementIndex, value); + timestamps.get(i).setData(0, time); + values.get(i).setData(0, value); elementIndex++; if (elementIndex == ARRAY_SIZE) { @@ -189,4 +204,17 @@ public class NVMBinaryTVList extends NVMTVList { // } // } } + + public static void main(String[] args) throws StartupException { + NVMSpaceManager.getInstance().init(); + + NVMBinaryTVList tvList = new NVMBinaryTVList("sg", "d0", "s0"); + int size = 5000; + for (int i = 0; i < size; i++) { + String v = String.valueOf(size - i); + tvList.putBinary(i, Binary.valueOf(v)); + } + + tvList.sort(); + } } diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMFloatTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMFloatTVList.java index b9814e8..2882aea 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMFloatTVList.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMFloatTVList.java @@ -2,6 +2,7 @@ package org.apache.iotdb.db.utils.datastructure; import static org.apache.iotdb.db.nvm.rescon.NVMPrimitiveArrayPool.ARRAY_SIZE; +import org.apache.iotdb.db.nvm.PerfMonitor; import org.apache.iotdb.db.nvm.space.NVMDataSpace; import org.apache.iotdb.db.rescon.PrimitiveArrayPool; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; @@ -160,9 +161,11 @@ public class NVMFloatTVList extends NVMTVList { int idx = 0; int length = time.length; + long startTime = System.currentTimeMillis(); for (int i = 0; i < length; i++) { putFloat(time[i], value[i]); } + PerfMonitor.add("NVMTVList.puts", System.currentTimeMillis() - startTime); // updateMinTimeAndSorted(time); // diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMTVList.java index 613f860..7bf854a 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMTVList.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMTVList.java @@ -165,10 +165,12 @@ public abstract class NVMTVList extends AbstractTVList { @Override protected void checkExpansion() { if ((size % ARRAY_SIZE) == 0) { + long time = System.currentTimeMillis(); NVMDataSpace valueSpace = expandValues(); NVMDataSpace timeSpace = NVMPrimitiveArrayPool.getInstance().getPrimitiveDataListByType(TSDataType.INT64, true); timestamps.add(timeSpace); NVMSpaceMetadataManager.getInstance().registerTVSpace(timeSpace, valueSpace, sgId, deviceId, measurementId); + PerfMonitor.add("NVMTVList.expand", System.currentTimeMillis() - time); } } @@ -236,28 +238,23 @@ public abstract class NVMTVList extends AbstractTVList { public void sort() { long time = System.currentTimeMillis(); initTempArrays(); -// System.out.println("init arr:" + (System.currentTimeMillis() - time)); PerfMonitor.add("sort-initarr", System.currentTimeMillis() - time); time = System.currentTimeMillis(); copyTVToTempArrays(); -// System.out.println("copy to arr:" + (System.currentTimeMillis() - time)); PerfMonitor.add("sort-copytoarr", System.currentTimeMillis() - time); time = System.currentTimeMillis(); sort(0, size); -// System.out.println("sort:" + (System.currentTimeMillis() - time)); PerfMonitor.add("sort-sort", System.currentTimeMillis() - time); time = System.currentTimeMillis(); copyTVFromTempArrays(); -// System.out.println("copy from arr:" + (System.currentTimeMillis() - time)); PerfMonitor.add("sort-copyfromarr", System.currentTimeMillis() - time); time = System.currentTimeMillis(); clearSortedValue(); clearSortedTime(); -// System.out.println("clear arr:" + (System.currentTimeMillis() - time)); PerfMonitor.add("sort-cleararr", System.currentTimeMillis() - time); sorted = true; diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java index a189e24..ea07fec 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java @@ -23,6 +23,7 @@ import static org.apache.iotdb.db.rescon.PrimitiveArrayPool.ARRAY_SIZE; import java.util.ArrayList; import java.util.List; +import org.apache.iotdb.db.nvm.PerfMonitor; import org.apache.iotdb.db.rescon.PrimitiveArrayPool; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; @@ -114,9 +115,11 @@ public abstract class TVList extends AbstractTVList { @Override protected void checkExpansion() { if ((size % ARRAY_SIZE) == 0) { + long time = System.currentTimeMillis(); expandValues(); timestamps.add((long[]) PrimitiveArrayPool.getInstance().getPrimitiveDataListByType( TSDataType.INT64)); + PerfMonitor.add("TVList.expand", System.currentTimeMillis() - time); } } diff --git a/server/src/main/java/org/apache/iotdb/db/writelog/io/MultiFileLogReader.java b/server/src/main/java/org/apache/iotdb/db/writelog/io/MultiFileLogReader.java index cf7b25f..75fb57b 100644 --- a/server/src/main/java/org/apache/iotdb/db/writelog/io/MultiFileLogReader.java +++ b/server/src/main/java/org/apache/iotdb/db/writelog/io/MultiFileLogReader.java @@ -22,6 +22,7 @@ package org.apache.iotdb.db.writelog.io; import java.io.File; import java.io.FileNotFoundException; import java.util.NoSuchElementException; +import org.apache.iotdb.db.nvm.PerfMonitor; import org.apache.iotdb.db.qp.physical.PhysicalPlan; /** @@ -47,6 +48,7 @@ public class MultiFileLogReader implements ILogReader { @Override public boolean hasNext() throws FileNotFoundException { + long time = System.currentTimeMillis(); if (files == null || files.length == 0) { return false; } @@ -54,14 +56,17 @@ public class MultiFileLogReader implements ILogReader { currentReader = new SingleFileLogReader(files[fileIdx++]); } if (currentReader.hasNext()) { + PerfMonitor.add("LogReader.hasNext", System.currentTimeMillis() - time); return true; } while (fileIdx < files.length) { currentReader.open(files[fileIdx++]); if (currentReader.hasNext()) { + PerfMonitor.add("LogReader.hasNext", System.currentTimeMillis() - time); return true; } } + PerfMonitor.add("LogReader.hasNext", System.currentTimeMillis() - time); return false; } @@ -70,6 +75,9 @@ public class MultiFileLogReader implements ILogReader { if (!hasNext()) { throw new NoSuchElementException(); } - return currentReader.next(); + long time = System.currentTimeMillis(); + PhysicalPlan plan = currentReader.next(); + PerfMonitor.add("LogReader.next", System.currentTimeMillis() - time); + return plan; } } diff --git a/server/src/main/java/org/apache/iotdb/db/writelog/recover/LogReplayer.java b/server/src/main/java/org/apache/iotdb/db/writelog/recover/LogReplayer.java index 9e8d8ec..52dc646 100644 --- a/server/src/main/java/org/apache/iotdb/db/writelog/recover/LogReplayer.java +++ b/server/src/main/java/org/apache/iotdb/db/writelog/recover/LogReplayer.java @@ -31,6 +31,7 @@ import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.engine.version.VersionController; import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.exception.storageGroup.StorageGroupProcessorException; +import org.apache.iotdb.db.nvm.PerfMonitor; import org.apache.iotdb.db.qp.physical.PhysicalPlan; import org.apache.iotdb.db.qp.physical.crud.BatchInsertPlan; import org.apache.iotdb.db.qp.physical.crud.DeletePlan; @@ -93,6 +94,7 @@ public class LogReplayer { ILogReader logReader = logNode.getLogReader(); try { while (logReader.hasNext()) { + long time = System.currentTimeMillis(); PhysicalPlan plan = logReader.next(); if (plan instanceof InsertPlan) { replayInsert((InsertPlan) plan); @@ -103,6 +105,7 @@ public class LogReplayer { } else if (plan instanceof BatchInsertPlan) { replayBatchInsert((BatchInsertPlan) plan); } + PerfMonitor.add("LogReplayer.replay", System.currentTimeMillis() - time); } } catch (IOException e) { throw new StorageGroupProcessorException("Cannot replay logs" + e.getMessage()); diff --git a/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java b/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java index 6e56541..8fbdb64 100644 --- a/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java +++ b/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java @@ -223,7 +223,7 @@ public class TsFileRecoverPerformer { long time = System.currentTimeMillis(); logReplayer.replayLogs(); - PerfMonitor.add("LogReplayer.replayLogs", System.currentTimeMillis() - time); + PerfMonitor.add("LogReplayer.replayLogs-ALL", System.currentTimeMillis() - time); flushRecoverdMemtable(recoverMemTable, restorableTsFileIOWriter); }
