This is an automated email from the ASF dual-hosted git repository. east pushed a commit to branch nvmlogging in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit fa5ec872fc6487c8bffbf6ac993a20cdb27e7314 Author: mdf369 <[email protected]> AuthorDate: Mon Jan 6 21:47:43 2020 +0800 update recovery tsDataMap (no StringSpace) --- .../iotdb/db/nvm/metadata/FreeSpaceBitMap.java | 8 +-- .../apache/iotdb/db/nvm/metadata/TSDataMap.java | 22 -------- .../iotdb/db/nvm/metadata/TimeValueMapper.java | 4 ++ .../db/nvm/metadata/TimeseriesTimeIndexMapper.java | 60 ++++++++++++++++++++++ .../nvm/recover/NVMMemtableRecoverPerformer.java | 2 +- .../apache/iotdb/db/nvm/space/NVMSpaceManager.java | 6 +++ .../db/nvm/space/NVMSpaceMetadataManager.java | 52 ++++++++----------- .../apache/iotdb/db/nvm/space/NVMStringSpace.java | 25 +++++++++ 8 files changed, 120 insertions(+), 59 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/nvm/metadata/FreeSpaceBitMap.java b/server/src/main/java/org/apache/iotdb/db/nvm/metadata/FreeSpaceBitMap.java index fb72503..9ae5cb9 100644 --- a/server/src/main/java/org/apache/iotdb/db/nvm/metadata/FreeSpaceBitMap.java +++ b/server/src/main/java/org/apache/iotdb/db/nvm/metadata/FreeSpaceBitMap.java @@ -1,7 +1,7 @@ package org.apache.iotdb.db.nvm.metadata; -import java.util.HashSet; -import java.util.Set; +import java.util.ArrayList; +import java.util.List; import org.apache.iotdb.db.nvm.space.NVMSpace; public class FreeSpaceBitMap extends NVMSpaceMetadata { @@ -14,8 +14,8 @@ public class FreeSpaceBitMap extends NVMSpaceMetadata { space.getByteBuffer().put(index, setFree ? (byte) 0 : (byte) 1); } - public Set<Integer> getValidSpaceIndexSet() { - Set<Integer> freeSpaceIndexList = new HashSet<>(); + public List<Integer> getValidSpaceIndexList() { + List<Integer> freeSpaceIndexList = new ArrayList<>(); for (int i = 0; i < space.getSize(); i++) { byte flag = space.getByteBuffer().get(i); if (flag == 1) { diff --git a/server/src/main/java/org/apache/iotdb/db/nvm/metadata/TSDataMap.java b/server/src/main/java/org/apache/iotdb/db/nvm/metadata/TSDataMap.java deleted file mode 100644 index 2f48fdd..0000000 --- a/server/src/main/java/org/apache/iotdb/db/nvm/metadata/TSDataMap.java +++ /dev/null @@ -1,22 +0,0 @@ -package org.apache.iotdb.db.nvm.metadata; - -import java.util.List; -import java.util.Map; -import org.apache.iotdb.db.nvm.space.NVMSpace; -import org.apache.iotdb.tsfile.utils.Pair; - -public class TSDataMap extends NVMSpaceMetadata { - - public TSDataMap(NVMSpace space) { - super(space); - } - - public void addSpaceToTimeSeries(int timeSpaceIndex, int valueSpaceIndex, String sgId, - String deviceId, String measurementId) { - // TODO - } - - public Map<String, Map<String, Map<String, Pair<List<Integer>, List<Integer>>>>> generateTSPathTVPairListMap() { - return null; - } -} diff --git a/server/src/main/java/org/apache/iotdb/db/nvm/metadata/TimeValueMapper.java b/server/src/main/java/org/apache/iotdb/db/nvm/metadata/TimeValueMapper.java index d08b7b9..9736b27 100644 --- a/server/src/main/java/org/apache/iotdb/db/nvm/metadata/TimeValueMapper.java +++ b/server/src/main/java/org/apache/iotdb/db/nvm/metadata/TimeValueMapper.java @@ -11,4 +11,8 @@ public class TimeValueMapper extends NVMSpaceMetadata { public void map(int timeSpaceIndex, int valueSpaceIndex) { space.getByteBuffer().putInt(timeSpaceIndex, valueSpaceIndex); } + + public int get(int timeSpaceIndex) { + return space.getByteBuffer().getInt(timeSpaceIndex); + } } diff --git a/server/src/main/java/org/apache/iotdb/db/nvm/metadata/TimeseriesTimeIndexMapper.java b/server/src/main/java/org/apache/iotdb/db/nvm/metadata/TimeseriesTimeIndexMapper.java new file mode 100644 index 0000000..0a7084d --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/nvm/metadata/TimeseriesTimeIndexMapper.java @@ -0,0 +1,60 @@ +package org.apache.iotdb.db.nvm.metadata; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; +import org.apache.iotdb.db.nvm.space.NVMSpace; +import org.apache.iotdb.db.nvm.space.NVMSpaceManager; +import org.apache.iotdb.db.nvm.space.NVMStringSpace; +import org.apache.iotdb.tsfile.utils.Pair; + +public class TimeseriesTimeIndexMapper extends NVMSpaceMetadata { + + // TODO + private final long STRING_SPACE_SIZE_MAX = 1000; + + private NVMStringSpace sgSapce; + private NVMStringSpace deviceSapce; + private NVMStringSpace measurementSapce; + + public TimeseriesTimeIndexMapper(NVMSpace space) throws IOException { + super(space); + + initTimeseriesSpaces(); + } + + private void initTimeseriesSpaces() throws IOException { + NVMSpaceManager spaceManager = NVMSpaceManager.getInstance(); + sgSapce = spaceManager.allocateStringSpace(STRING_SPACE_SIZE_MAX); + deviceSapce = spaceManager.allocateStringSpace(STRING_SPACE_SIZE_MAX); + measurementSapce = spaceManager.allocateStringSpace(STRING_SPACE_SIZE_MAX); + } + + public void mapTimeIndexToTimeSeries(int timeSpaceIndex, String sgId, + String deviceId, String measurementId) { + int sgIndex = sgSapce.put(sgId); + int deviceIndex = deviceSapce.put(sgId); + int measurementIndex = measurementSapce.put(sgId); + + mapTimeIndexToTimeSeries(timeSpaceIndex, sgIndex, deviceIndex, measurementIndex); + } + + private void mapTimeIndexToTimeSeries(int timeSpaceIndex, int sgIndex, int deviceIndex, int measurementIndex) { + int index = timeSpaceIndex * 3; + ByteBuffer byteBuffer = space.getByteBuffer(); + byteBuffer.putInt(index, sgIndex); + byteBuffer.putInt(index + 1, deviceIndex); + byteBuffer.putInt(index + 2, measurementIndex); + } + + public String[] getTimeseries(int timeSpaceIndex) { + ByteBuffer byteBuffer = space.getByteBuffer(); + int index = timeSpaceIndex * 3; + String[] timeseries = new String[3]; + timeseries[0] = sgSapce.get(byteBuffer.getInt(index)); + timeseries[1] = deviceSapce.get(byteBuffer.getInt(index + 1)); + timeseries[2] = measurementSapce.get(byteBuffer.getInt(index + 2)); + return timeseries; + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/nvm/recover/NVMMemtableRecoverPerformer.java b/server/src/main/java/org/apache/iotdb/db/nvm/recover/NVMMemtableRecoverPerformer.java index 273796d..7e43b18 100644 --- a/server/src/main/java/org/apache/iotdb/db/nvm/recover/NVMMemtableRecoverPerformer.java +++ b/server/src/main/java/org/apache/iotdb/db/nvm/recover/NVMMemtableRecoverPerformer.java @@ -38,7 +38,7 @@ public class NVMMemtableRecoverPerformer { private Map<String, Map<String, Map<String, Pair<List<NVMDataSpace>, List<NVMDataSpace>>>>> recoverDataInNVM() throws IOException { Map<String, Map<String, Map<String, Pair<List<NVMDataSpace>, List<NVMDataSpace>>>>> dataMap = new HashMap<>(); - Map<String, Map<String, Map<String, Pair<List<Integer>, List<Integer>>>>> indexMap = NVMSpaceMetadataManager.getInstance().getValidSpaceIndexMap(); + Map<String, Map<String, Map<String, Pair<List<Integer>, List<Integer>>>>> indexMap = NVMSpaceMetadataManager.getInstance().getTimeseriesTVIndexMap(); List<NVMDataSpace> dataList = NVMSpaceManager.getInstance().getAllNVMData(); for (Entry<String, Map<String, Map<String, Pair<List<Integer>, List<Integer>>>>> sgIndexEntry : indexMap diff --git a/server/src/main/java/org/apache/iotdb/db/nvm/space/NVMSpaceManager.java b/server/src/main/java/org/apache/iotdb/db/nvm/space/NVMSpaceManager.java index 5a5b7e6..3da68bf 100644 --- a/server/src/main/java/org/apache/iotdb/db/nvm/space/NVMSpaceManager.java +++ b/server/src/main/java/org/apache/iotdb/db/nvm/space/NVMSpaceManager.java @@ -92,6 +92,12 @@ public class NVMSpaceManager { return nvmSpace; } + public synchronized NVMStringSpace allocateStringSpace(long size) throws IOException { + NVMStringSpace nvmSpace = new NVMStringSpace(curOffset, size, nvmFileChannel.map(MAP_MODE, curOffset, size)); + curOffset += size; + return nvmSpace; + } + public synchronized NVMDataSpace allocateDataSpace(long size, TSDataType dataType) { checkIsFull(); diff --git a/server/src/main/java/org/apache/iotdb/db/nvm/space/NVMSpaceMetadataManager.java b/server/src/main/java/org/apache/iotdb/db/nvm/space/NVMSpaceMetadataManager.java index 8863408..93f3d99 100644 --- a/server/src/main/java/org/apache/iotdb/db/nvm/space/NVMSpaceMetadataManager.java +++ b/server/src/main/java/org/apache/iotdb/db/nvm/space/NVMSpaceMetadataManager.java @@ -3,6 +3,8 @@ package org.apache.iotdb.db.nvm.space; import static org.apache.iotdb.db.nvm.space.NVMSpaceManager.NVMSPACE_NUM_MAX; import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -10,7 +12,7 @@ import java.util.TreeSet; import org.apache.iotdb.db.nvm.metadata.DataTypeMemo; import org.apache.iotdb.db.nvm.metadata.FreeSpaceBitMap; import org.apache.iotdb.db.nvm.metadata.SpaceCount; -import org.apache.iotdb.db.nvm.metadata.TSDataMap; +import org.apache.iotdb.db.nvm.metadata.TimeseriesTimeIndexMapper; import org.apache.iotdb.db.nvm.metadata.TimeValueMapper; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.utils.Pair; @@ -29,7 +31,7 @@ public class NVMSpaceMetadataManager { private FreeSpaceBitMap freeSpaceBitMap; private DataTypeMemo dataTypeMemo; private TimeValueMapper timeValueMapper; - private TSDataMap tsDataMap; + private TimeseriesTimeIndexMapper timeseriesTimeIndexMapper; private NVMSpaceManager spaceManager = NVMSpaceManager.getInstance(); @@ -40,7 +42,7 @@ public class NVMSpaceMetadataManager { freeSpaceBitMap = new FreeSpaceBitMap(spaceManager.allocateSpace(BITMAP_FIELD_BYTE_SIZE)); dataTypeMemo = new DataTypeMemo(spaceManager.allocateSpace(DATATYPE_FIELD_BYTE_SIZE)); timeValueMapper = new TimeValueMapper(spaceManager.allocateSpace(TVMAP_FIELD_BYTE_SIZE)); - tsDataMap = new TSDataMap(spaceManager.allocateSpace(TIMESERIES_FIELD_BYTE_SIZE)); + timeseriesTimeIndexMapper = new TimeseriesTimeIndexMapper(spaceManager.allocateSpace(TIMESERIES_FIELD_BYTE_SIZE)); } public static NVMSpaceMetadataManager getInstance() { @@ -64,42 +66,28 @@ public class NVMSpaceMetadataManager { dataTypeMemo.set(valueSpaceIndex, valueSpace.getDataType()); timeValueMapper.map(timeSpaceIndex, valueSpaceIndex); - tsDataMap.addSpaceToTimeSeries(timeSpaceIndex, valueSpaceIndex, sgId, deviceId, measurementId); + timeseriesTimeIndexMapper + .mapTimeIndexToTimeSeries(timeSpaceIndex, sgId, deviceId, measurementId); } public void unregisterSpace(NVMDataSpace space) { freeSpaceBitMap.update(space.getIndex(), true); } - public Map<String, Map<String, Map<String, Pair<List<Integer>, List<Integer>>>>> getValidSpaceIndexMap() { - Set<Integer> validSpaceIndexSet = freeSpaceBitMap.getValidSpaceIndexSet(); - Map<String, Map<String, Map<String, Pair<List<Integer>, List<Integer>>>>> tsTVMap = tsDataMap.generateTSPathTVPairListMap(); - for (Map<String, Map<String, Pair<List<Integer>, List<Integer>>>> dmTVMap : tsTVMap.values()) { - for (Map<String, Pair<List<Integer>, List<Integer>>> mTVMap : dmTVMap.values()) { - for (Pair<List<Integer>, List<Integer>> tvIndexListPair : mTVMap.values()) { - List<Integer> timeIndexList = tvIndexListPair.left; - List<Integer> valueIndexList = tvIndexListPair.right; - - Set<Integer> toBeRemovedIndexSet = new TreeSet<>((o1, o2) -> o2 - o1); - for (int i = 0; i < timeIndexList.size(); i++) { - if (!validSpaceIndexSet.contains(timeIndexList.get(i))) { - toBeRemovedIndexSet.add(i); - } - } - for (int i = 0; i < timeIndexList.size(); i++) { - if (!validSpaceIndexSet.contains(timeIndexList.get(i))) { - toBeRemovedIndexSet.add(i); - } - } - - for (Integer index : toBeRemovedIndexSet) { - timeIndexList.remove(index); - valueIndexList.remove(index); - } - } - } + public Map<String, Map<String, Map<String, Pair<List<Integer>, List<Integer>>>>> getTimeseriesTVIndexMap() { + Map<String, Map<String, Map<String, Pair<List<Integer>, List<Integer>>>>> tsTVIndexMap = new HashMap<>(); + List<Integer> validSpaceIndexList = freeSpaceBitMap.getValidSpaceIndexList(); + for (Integer timeSpaceIndex : validSpaceIndexList) { + int valueSpaceIndex = timeValueMapper.get(timeSpaceIndex); + String[] timeseries = timeseriesTimeIndexMapper.getTimeseries(timeSpaceIndex); + + Map<String, Map<String, Pair<List<Integer>, List<Integer>>>> deviceTVMap = tsTVIndexMap.computeIfAbsent(timeseries[0], k -> new HashMap<>()); + Map<String, Pair<List<Integer>, List<Integer>>> measurementTVMap = deviceTVMap.computeIfAbsent(timeseries[1], k -> new HashMap<>()); + Pair<List<Integer>, List<Integer>> tvPairList = measurementTVMap.computeIfAbsent(timeseries[2], k -> new Pair<>(new ArrayList<>(), new ArrayList<>())); + tvPairList.left.add(timeSpaceIndex); + tvPairList.right.add(valueSpaceIndex); } - return tsTVMap; + return tsTVIndexMap; } public List<TSDataType> getDataTypeList(int count) { diff --git a/server/src/main/java/org/apache/iotdb/db/nvm/space/NVMStringSpace.java b/server/src/main/java/org/apache/iotdb/db/nvm/space/NVMStringSpace.java new file mode 100644 index 0000000..d6c1fc0 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/nvm/space/NVMStringSpace.java @@ -0,0 +1,25 @@ +package org.apache.iotdb.db.nvm.space; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +public class NVMStringSpace extends NVMSpace { + + List<String> existStringList = new ArrayList<>(); + + NVMStringSpace(long offset, long size, ByteBuffer byteBuffer) { + super(offset, size, byteBuffer); + + // TODO recover set + } + + public int put(String s) { + // TODO + } + + public String get(int index) { + // TODO + return null; + } +}
