This is an automated email from the ASF dual-hosted git repository. qiaojialin pushed a commit to branch vector_review in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 610586eee19b1600edba4d85d73aac75f5154bc4 Author: qiaojialin <[email protected]> AuthorDate: Thu Apr 8 20:59:30 2021 +0800 add javadoc --- .../org/apache/iotdb/VectorSessionExample.java | 28 +-- .../iotdb/db/engine/flush/MemTableFlushTask.java | 53 +++--- .../iotdb/db/engine/memtable/AbstractMemTable.java | 29 ++-- .../db/engine/memtable/IWritableMemChunk.java | 10 +- .../db/engine/storagegroup/TsFileProcessor.java | 26 +-- .../iotdb/db/utils/datastructure/VectorTVList.java | 188 ++++++--------------- .../java/org/apache/iotdb/session/Session.java | 2 +- .../org/apache/iotdb/session/SessionUtils.java | 21 +-- .../java/org/apache/iotdb/tsfile/utils/Binary.java | 2 + .../java/org/apache/iotdb/tsfile/utils/BitMap.java | 12 +- .../apache/iotdb/tsfile/write/record/Tablet.java | 14 +- .../tsfile/write/schema/MeasurementSchema.java | 7 - .../write/schema/VectorMeasurementSchema.java | 31 ++-- .../org/apache/iotdb/tsfile/utils/BitMapTest.java | 22 +-- 14 files changed, 175 insertions(+), 270 deletions(-) diff --git a/example/session/src/main/java/org/apache/iotdb/VectorSessionExample.java b/example/session/src/main/java/org/apache/iotdb/VectorSessionExample.java index 7525cae..7b03b6e 100644 --- a/example/session/src/main/java/org/apache/iotdb/VectorSessionExample.java +++ b/example/session/src/main/java/org/apache/iotdb/VectorSessionExample.java @@ -97,30 +97,30 @@ public class VectorSessionExample { private static void createTemplate() throws StatementExecutionException, IoTDBConnectionException { List<List<String>> measurementList = new ArrayList<>(); - List<String> measurements = new ArrayList<>(); + List<String> vectorMeasurement = new ArrayList<>(); for (int i = 1; i <= 2; i++) { - measurements.add("s" + i); + vectorMeasurement.add("s" + i); } - measurementList.add(measurements); + measurementList.add(vectorMeasurement); List<List<TSDataType>> dataTypeList = new ArrayList<>(); - List<TSDataType> dataTypes = new ArrayList<>(); - dataTypes.add(TSDataType.INT64); - dataTypes.add(TSDataType.INT32); - dataTypeList.add(dataTypes); + List<TSDataType> vectorDatatype = new ArrayList<>(); + vectorDatatype.add(TSDataType.INT64); + vectorDatatype.add(TSDataType.INT32); + dataTypeList.add(vectorDatatype); List<List<TSEncoding>> encodingList = new ArrayList<>(); - List<TSEncoding> encodings = new ArrayList<>(); + List<TSEncoding> vectorEncoding = new ArrayList<>(); for (int i = 1; i <= 2; i++) { - encodings.add(TSEncoding.RLE); + vectorEncoding.add(TSEncoding.RLE); } - encodingList.add(encodings); + encodingList.add(vectorEncoding); - List<CompressionType> compressionTypes = new ArrayList<>(); - compressionTypes.add(CompressionType.SNAPPY); + List<CompressionType> compressionTypeList = new ArrayList<>(); + compressionTypeList.add(CompressionType.SNAPPY); session.createDeviceTemplate( - "template1", measurementList, dataTypeList, encodingList, compressionTypes); + "template1", measurementList, dataTypeList, encodingList, compressionTypeList); session.setDeviceTemplate("template1", "root.sg_1"); } @@ -190,7 +190,7 @@ public class VectorSessionExample { sensors[row] = new Random().nextInt(); if (time % 5 == 0) { - bitMaps[1].mark((int) row); + bitMaps[1].mark(row); } if (tablet.rowSize == tablet.getMaxRowNumber()) { diff --git a/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java b/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java index 013ce82..8206f7f 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java @@ -161,68 +161,81 @@ public class MemTableFlushTask { new Runnable() { private void writeOneSeries( TVList tvPairs, IChunkWriter seriesWriterImpl, TSDataType dataType) { - for (int i = 0; i < tvPairs.size(); i++) { - long time = tvPairs.getTime(i); + for (int sortedRowIndex = 0; sortedRowIndex < tvPairs.size(); sortedRowIndex++) { + long time = tvPairs.getTime(sortedRowIndex); // skip duplicated data - if ((i + 1 < tvPairs.size() && (time == tvPairs.getTime(i + 1)))) { + if ((sortedRowIndex + 1 < tvPairs.size() + && (time == tvPairs.getTime(sortedRowIndex + 1)))) { continue; } // store last point for SDT - if (dataType != TSDataType.VECTOR && i + 1 == tvPairs.size()) { + if (dataType != TSDataType.VECTOR && sortedRowIndex + 1 == tvPairs.size()) { ((ChunkWriterImpl) seriesWriterImpl).setLastPoint(true); } switch (dataType) { case BOOLEAN: - seriesWriterImpl.write(time, tvPairs.getBoolean(i), false); + seriesWriterImpl.write(time, tvPairs.getBoolean(sortedRowIndex), false); break; case INT32: - seriesWriterImpl.write(time, tvPairs.getInt(i), false); + seriesWriterImpl.write(time, tvPairs.getInt(sortedRowIndex), false); break; case INT64: - seriesWriterImpl.write(time, tvPairs.getLong(i), false); + seriesWriterImpl.write(time, tvPairs.getLong(sortedRowIndex), false); break; case FLOAT: - seriesWriterImpl.write(time, tvPairs.getFloat(i), false); + seriesWriterImpl.write(time, tvPairs.getFloat(sortedRowIndex), false); break; case DOUBLE: - seriesWriterImpl.write(time, tvPairs.getDouble(i), false); + seriesWriterImpl.write(time, tvPairs.getDouble(sortedRowIndex), false); break; case TEXT: - seriesWriterImpl.write(time, tvPairs.getBinary(i), false); + seriesWriterImpl.write(time, tvPairs.getBinary(sortedRowIndex), false); break; case VECTOR: VectorTVList vectorTVPairs = (VectorTVList) tvPairs; List<TSDataType> dataTypes = vectorTVPairs.getTsDataTypes(); - int index = vectorTVPairs.getValueIndex(i); - for (int j = 0; j < dataTypes.size(); j++) { - boolean isNull = vectorTVPairs.isValueMarked(index, j); - switch (dataTypes.get(j)) { + int originRowIndex = vectorTVPairs.getValueIndex(sortedRowIndex); + for (int columnIndex = 0; columnIndex < dataTypes.size(); columnIndex++) { + boolean isNull = vectorTVPairs.isValueMarked(originRowIndex, columnIndex); + switch (dataTypes.get(columnIndex)) { case BOOLEAN: seriesWriterImpl.write( - time, vectorTVPairs.getBooleanByValueIndex(index, j), isNull); + time, + vectorTVPairs.getBooleanByValueIndex(originRowIndex, columnIndex), + isNull); break; case INT32: seriesWriterImpl.write( - time, vectorTVPairs.getIntByValueIndex(index, j), isNull); + time, + vectorTVPairs.getIntByValueIndex(originRowIndex, columnIndex), + isNull); break; case INT64: seriesWriterImpl.write( - time, vectorTVPairs.getLongByValueIndex(index, j), isNull); + time, + vectorTVPairs.getLongByValueIndex(originRowIndex, columnIndex), + isNull); break; case FLOAT: seriesWriterImpl.write( - time, vectorTVPairs.getFloatByValueIndex(index, j), isNull); + time, + vectorTVPairs.getFloatByValueIndex(originRowIndex, columnIndex), + isNull); break; case DOUBLE: seriesWriterImpl.write( - time, vectorTVPairs.getDoubleByValueIndex(index, j), isNull); + time, + vectorTVPairs.getDoubleByValueIndex(originRowIndex, columnIndex), + isNull); break; case TEXT: seriesWriterImpl.write( - time, vectorTVPairs.getBinaryByValueIndex(index, j), isNull); + time, + vectorTVPairs.getBinaryByValueIndex(originRowIndex, columnIndex), + isNull); break; default: LOGGER.error( diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java index a621ed7..e665822 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java @@ -21,7 +21,6 @@ package org.apache.iotdb.db.engine.memtable; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk; import org.apache.iotdb.db.exception.WriteProcessException; -import org.apache.iotdb.db.exception.metadata.MetadataException; import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.metadata.MetaUtils; import org.apache.iotdb.db.metadata.PartialPath; @@ -295,40 +294,42 @@ public abstract class AbstractMemTable implements IMemTable { public ReadOnlyMemChunk query( String deviceId, String measurement, - IMeasurementSchema schema, + IMeasurementSchema partialVectorSchema, long timeLowerBound, List<TimeRange> deletionList) - throws IOException, QueryProcessException, MetadataException { - if (schema.getType() == TSDataType.VECTOR) { + throws IOException, QueryProcessException { + if (partialVectorSchema.getType() == TSDataType.VECTOR) { if (!memTableMap.containsKey(deviceId)) { return null; } - IWritableMemChunk memChunk = memTableMap.get(deviceId).get(schema.getMeasurementId()); + IWritableMemChunk vectorMemChunk = + memTableMap.get(deviceId).get(partialVectorSchema.getMeasurementId()); - List<String> measurementIdList = schema.getValueMeasurementIdList(); + List<String> measurementIdList = partialVectorSchema.getValueMeasurementIdList(); List<Integer> columns = new ArrayList<>(); - IMeasurementSchema vectorSchema = memChunk.getSchema(); + IMeasurementSchema vectorSchema = vectorMemChunk.getSchema(); for (String queryingMeasurement : measurementIdList) { columns.add(vectorSchema.getValueMeasurementIdList().indexOf(queryingMeasurement)); } // get sorted tv list is synchronized so different query can get right sorted list reference - TVList chunkCopy = memChunk.getSortedTVListForQuery(columns); - int curSize = chunkCopy.size(); - return new ReadOnlyMemChunk(schema, chunkCopy, curSize, deletionList); + TVList vectorTVListCopy = vectorMemChunk.getSortedTVListForQuery(columns); + int curSize = vectorTVListCopy.size(); + return new ReadOnlyMemChunk(partialVectorSchema, vectorTVListCopy, curSize, deletionList); } else { if (!checkPath(deviceId, measurement)) { return null; } - IWritableMemChunk memChunk = memTableMap.get(deviceId).get(schema.getMeasurementId()); + IWritableMemChunk memChunk = + memTableMap.get(deviceId).get(partialVectorSchema.getMeasurementId()); // get sorted tv list is synchronized so different query can get right sorted list reference TVList chunkCopy = memChunk.getSortedTVListForQuery(); int curSize = chunkCopy.size(); return new ReadOnlyMemChunk( measurement, - schema.getType(), - schema.getEncodingType(), + partialVectorSchema.getType(), + partialVectorSchema.getEncodingType(), chunkCopy, - schema.getProps(), + partialVectorSchema.getProps(), curSize, deletionList); } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunk.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunk.java index 2b8c5a6..283a0da 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunk.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunk.java @@ -83,17 +83,11 @@ public interface IWritableMemChunk { TVList getSortedTVListForQuery(); /** - * served for query requests. - * - * <p>if tv list has been sorted, just return reference of it - * - * <p>if tv list hasn't been sorted and has no reference, sort and return reference of it - * - * <p>if tv list hasn't been sorted and has reference we should copy and sort it, then return ths - * list + * served for vector query requests. * * <p>the mechanism is just like copy on write * + * @param columnIndexList indices of queried columns in the full VectorTVList * @return sorted tv list */ TVList getSortedTVListForQuery(List<Integer> columnIndexList); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java index dbd6464..352919e 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java @@ -285,12 +285,12 @@ public class TsFileProcessor { long chunkMetadataIncrement = 0L; String deviceId = insertRowPlan.getDeviceId().getFullPath(); long unsealedResourceIncrement = tsFileResource.estimateRamIncrement(deviceId); - int columnCount = 0; + int columnIndex = 0; for (int i = 0; i < insertRowPlan.getMeasurementMNodes().length; i++) { // skip failed Measurements - if (insertRowPlan.getDataTypes()[columnCount] == null + if (insertRowPlan.getDataTypes()[columnIndex] == null || insertRowPlan.getMeasurements()[i] == null) { - columnCount++; + columnIndex++; continue; } if (workMemTable.checkIfChunkDoesNotExist(deviceId, insertRowPlan.getMeasurements()[i])) { @@ -306,8 +306,8 @@ public class TsFileProcessor { } else { chunkMetadataIncrement += ChunkMetadata.calculateRamSize( - insertRowPlan.getMeasurements()[i], insertRowPlan.getDataTypes()[columnCount]); - memTableIncrement += TVList.tvListArrayMemSize(insertRowPlan.getDataTypes()[columnCount]); + insertRowPlan.getMeasurements()[i], insertRowPlan.getDataTypes()[columnIndex]); + memTableIncrement += TVList.tvListArrayMemSize(insertRowPlan.getDataTypes()[columnIndex]); } } else { // here currentChunkPointNum >= 1 @@ -315,13 +315,13 @@ public class TsFileProcessor { workMemTable.getCurrentChunkPointNum(deviceId, insertRowPlan.getMeasurements()[i]); memTableIncrement += (currentChunkPointNum % PrimitiveArrayManager.ARRAY_SIZE) == 0 - ? TVList.tvListArrayMemSize(insertRowPlan.getDataTypes()[columnCount]) + ? TVList.tvListArrayMemSize(insertRowPlan.getDataTypes()[columnIndex]) : 0; } // TEXT data mem size - if (insertRowPlan.getDataTypes()[columnCount] == TSDataType.TEXT) { + if (insertRowPlan.getDataTypes()[columnIndex] == TSDataType.TEXT) { textDataIncrement += - MemUtils.getBinarySize((Binary) insertRowPlan.getValues()[columnCount]); + MemUtils.getBinarySize((Binary) insertRowPlan.getValues()[columnIndex]); } } updateMemoryInfo( @@ -338,7 +338,7 @@ public class TsFileProcessor { String deviceId = insertTabletPlan.getDeviceId().getFullPath(); long unsealedResourceIncrement = tsFileResource.estimateRamIncrement(deviceId); - int columnCount = 0; + int columnIndex = 0; for (int i = 0; i < insertTabletPlan.getMeasurementMNodes().length; i++) { // for aligned timeseries if (insertTabletPlan.getMeasurementMNodes()[i].getSchema().getType() == TSDataType.VECTOR) { @@ -346,17 +346,17 @@ public class TsFileProcessor { (VectorMeasurementSchema) insertTabletPlan.getMeasurementMNodes()[i].getSchema(); Object[] columns = new Object[vectorSchema.getValueMeasurementIdList().size()]; for (int j = 0; j < vectorSchema.getValueMeasurementIdList().size(); j++) { - columns[j] = insertTabletPlan.getColumns()[columnCount++]; + columns[j] = insertTabletPlan.getColumns()[columnIndex++]; } updateVectorMemCost(vectorSchema, deviceId, start, end, memIncrements, columns); } // for non aligned else { // skip failed Measurements - TSDataType dataType = insertTabletPlan.getDataTypes()[columnCount]; + TSDataType dataType = insertTabletPlan.getDataTypes()[columnIndex]; String measurement = insertTabletPlan.getMeasurements()[i]; - Object column = insertTabletPlan.getColumns()[columnCount]; - columnCount++; + Object column = insertTabletPlan.getColumns()[columnIndex]; + columnIndex++; if (dataType == null || column == null || measurement == null) { continue; } diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/VectorTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/VectorTVList.java index eff7b4b..ee4145a 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/VectorTVList.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/VectorTVList.java @@ -73,7 +73,7 @@ public class VectorTVList extends TVList { switch (dataTypes.get(i)) { case TEXT: ((Binary[]) columnValues.get(arrayIndex))[elementIndex] = - columnValue != null ? (Binary) columnValue : new Binary(""); + columnValue != null ? (Binary) columnValue : Binary.EMPTY_VALUE; break; case FLOAT: ((float[]) columnValues.get(arrayIndex))[elementIndex] = @@ -93,7 +93,7 @@ public class VectorTVList extends TVList { break; case BOOLEAN: ((boolean[]) columnValues.get(arrayIndex))[elementIndex] = - columnValue != null ? (boolean) columnValue : false; + columnValue != null && (boolean) columnValue; break; default: break; @@ -128,7 +128,7 @@ public class VectorTVList extends TVList { List<Object> columnValues = values.get(i); if (bitMaps != null && bitMaps.get(i) != null - && bitMaps.get(i).get(arrayIndex).get(elementIndex)) { + && bitMaps.get(i).get(arrayIndex).isMarked(elementIndex)) { continue; } switch (dataTypes.get(i)) { @@ -196,155 +196,64 @@ public class VectorTVList extends TVList { return vectorTVList; } - public int getInt(int index, int column) { - if (index >= size) { - throw new ArrayIndexOutOfBoundsException(index); - } - int arrayIndex = index / ARRAY_SIZE; - int elementIndex = index % ARRAY_SIZE; - int valueIndex = indices.get(arrayIndex)[elementIndex]; - return getIntByValueIndex(valueIndex, column); - } - - public int getIntByValueIndex(int valueIndex, int column) { - if (dataTypes.get(column) != TSDataType.INT32) { - throw new UnsupportedOperationException(ERR_DATATYPE_NOT_CONSISTENT); - } - if (valueIndex >= size) { - throw new ArrayIndexOutOfBoundsException(valueIndex); - } - int arrayIndex = valueIndex / ARRAY_SIZE; - int elementIndex = valueIndex % ARRAY_SIZE; - List<Object> columnValues = values.get(column); + /** + * @param rowIndex value index inside this column + * @param columnIndex index of the column + * @return + */ + public int getIntByValueIndex(int rowIndex, int columnIndex) { + int arrayIndex = rowIndex / ARRAY_SIZE; + int elementIndex = rowIndex % ARRAY_SIZE; + List<Object> columnValues = values.get(columnIndex); return ((int[]) columnValues.get(arrayIndex))[elementIndex]; } - public long getLong(int index, int column) { - if (index >= size) { - throw new ArrayIndexOutOfBoundsException(index); - } - int arrayIndex = index / ARRAY_SIZE; - int elementIndex = index % ARRAY_SIZE; - int valueIndex = indices.get(arrayIndex)[elementIndex]; - return getLongByValueIndex(valueIndex, column); - } - - public long getLongByValueIndex(int valueIndex, int column) { - if (dataTypes.get(column) != TSDataType.INT64) { - throw new UnsupportedOperationException(ERR_DATATYPE_NOT_CONSISTENT); - } - if (valueIndex >= size) { - throw new ArrayIndexOutOfBoundsException(valueIndex); - } - int arrayIndex = valueIndex / ARRAY_SIZE; - int elementIndex = valueIndex % ARRAY_SIZE; - List<Object> columnValues = values.get(column); + public long getLongByValueIndex(int rowIndex, int columnIndex) { + int arrayIndex = rowIndex / ARRAY_SIZE; + int elementIndex = rowIndex % ARRAY_SIZE; + List<Object> columnValues = values.get(columnIndex); return ((long[]) columnValues.get(arrayIndex))[elementIndex]; } - public float getFloat(int index, int column) { - if (index >= size) { - throw new ArrayIndexOutOfBoundsException(index); - } - int arrayIndex = index / ARRAY_SIZE; - int elementIndex = index % ARRAY_SIZE; - int valueIndex = indices.get(arrayIndex)[elementIndex]; - return getFloatByValueIndex(valueIndex, column); - } - - public float getFloatByValueIndex(int valueIndex, int column) { - if (dataTypes.get(column) != TSDataType.FLOAT) { - throw new UnsupportedOperationException(ERR_DATATYPE_NOT_CONSISTENT); - } - if (valueIndex >= size) { - throw new ArrayIndexOutOfBoundsException(valueIndex); - } - int arrayIndex = valueIndex / ARRAY_SIZE; - int elementIndex = valueIndex % ARRAY_SIZE; - List<Object> columnValues = values.get(column); + public float getFloatByValueIndex(int rowIndex, int columnIndex) { + int arrayIndex = rowIndex / ARRAY_SIZE; + int elementIndex = rowIndex % ARRAY_SIZE; + List<Object> columnValues = values.get(columnIndex); return ((float[]) columnValues.get(arrayIndex))[elementIndex]; } - public double getDouble(int index, int column) { - if (index >= size) { - throw new ArrayIndexOutOfBoundsException(index); - } - int arrayIndex = index / ARRAY_SIZE; - int elementIndex = index % ARRAY_SIZE; - int valueIndex = indices.get(arrayIndex)[elementIndex]; - return getDoubleByValueIndex(valueIndex, column); - } - - public double getDoubleByValueIndex(int valueIndex, int column) { - if (dataTypes.get(column) != TSDataType.DOUBLE) { - throw new UnsupportedOperationException(ERR_DATATYPE_NOT_CONSISTENT); - } - if (valueIndex >= size) { - throw new ArrayIndexOutOfBoundsException(valueIndex); - } - int arrayIndex = valueIndex / ARRAY_SIZE; - int elementIndex = valueIndex % ARRAY_SIZE; - List<Object> columnValues = values.get(column); + public double getDoubleByValueIndex(int rowIndex, int columnIndex) { + int arrayIndex = rowIndex / ARRAY_SIZE; + int elementIndex = rowIndex % ARRAY_SIZE; + List<Object> columnValues = values.get(columnIndex); return ((double[]) columnValues.get(arrayIndex))[elementIndex]; } - public Binary getBinary(int index, int column) { - if (index >= size) { - throw new ArrayIndexOutOfBoundsException(index); - } - int arrayIndex = index / ARRAY_SIZE; - int elementIndex = index % ARRAY_SIZE; - int valueIndex = indices.get(arrayIndex)[elementIndex]; - return getBinaryByValueIndex(valueIndex, column); - } - - public Binary getBinaryByValueIndex(int valueIndex, int column) { - if (dataTypes.get(column) != TSDataType.TEXT) { - throw new UnsupportedOperationException(ERR_DATATYPE_NOT_CONSISTENT); - } - if (valueIndex >= size) { - throw new ArrayIndexOutOfBoundsException(valueIndex); - } - int arrayIndex = valueIndex / ARRAY_SIZE; - int elementIndex = valueIndex % ARRAY_SIZE; - List<Object> columnValues = values.get(column); + public Binary getBinaryByValueIndex(int rowIndex, int columnIndex) { + int arrayIndex = rowIndex / ARRAY_SIZE; + int elementIndex = rowIndex % ARRAY_SIZE; + List<Object> columnValues = values.get(columnIndex); return ((Binary[]) columnValues.get(arrayIndex))[elementIndex]; } - public boolean getBoolean(int index, int column) { - if (index >= size) { - throw new ArrayIndexOutOfBoundsException(index); - } - int arrayIndex = index / ARRAY_SIZE; - int elementIndex = index % ARRAY_SIZE; - int valueIndex = indices.get(arrayIndex)[elementIndex]; - return getBooleanByValueIndex(valueIndex, column); - } - - public boolean getBooleanByValueIndex(int valueIndex, int column) { - if (dataTypes.get(column) != TSDataType.BOOLEAN) { - throw new UnsupportedOperationException(ERR_DATATYPE_NOT_CONSISTENT); - } - if (valueIndex >= size) { - throw new ArrayIndexOutOfBoundsException(valueIndex); - } - int arrayIndex = valueIndex / ARRAY_SIZE; - int elementIndex = valueIndex % ARRAY_SIZE; - List<Object> columnValues = values.get(column); + public boolean getBooleanByValueIndex(int rowIndex, int columnIndex) { + int arrayIndex = rowIndex / ARRAY_SIZE; + int elementIndex = rowIndex % ARRAY_SIZE; + List<Object> columnValues = values.get(columnIndex); return ((boolean[]) columnValues.get(arrayIndex))[elementIndex]; } - public boolean isValueMarked(int valueIndex, int column) { - if (valueIndex >= size) { + public boolean isValueMarked(int rowIndex, int columnIndex) { + if (rowIndex >= size) { return false; } - if (bitMaps == null || bitMaps.get(column) == null) { + if (bitMaps == null || bitMaps.get(columnIndex) == null) { return false; } - int arrayIndex = valueIndex / ARRAY_SIZE; - int elementIndex = valueIndex % ARRAY_SIZE; - List<BitMap> columnBitMaps = bitMaps.get(column); - return columnBitMaps.get(arrayIndex).get(elementIndex); + int arrayIndex = rowIndex / ARRAY_SIZE; + int elementIndex = rowIndex % ARRAY_SIZE; + List<BitMap> columnBitMaps = bitMaps.get(columnIndex); + return columnBitMaps.get(arrayIndex).isMarked(elementIndex); } public List<List<Object>> getValues() { @@ -355,18 +264,11 @@ public class VectorTVList extends TVList { return dataTypes; } - public List<int[]> getIndices() { - return indices; - } - - protected void set(int index, long timestamp, int valueIndex) { - if (index >= size) { - throw new ArrayIndexOutOfBoundsException(index); - } + protected void set(int index, long timestamp, int value) { int arrayIndex = index / ARRAY_SIZE; int elementIndex = index % ARRAY_SIZE; timestamps.get(arrayIndex)[elementIndex] = timestamp; - indices.get(arrayIndex)[elementIndex] = valueIndex; + indices.get(arrayIndex)[elementIndex] = value; } @Override @@ -601,7 +503,7 @@ public class VectorTVList extends TVList { indices.get(arrayIdx)[elementIdx + i] = size; if (bitMaps != null) { for (int j = 0; j < bitMaps.length; j++) { - if (bitMaps[j] != null && bitMaps[j].get(idx + i)) { + if (bitMaps[j] != null && bitMaps[j].isMarked(idx + i)) { markNullValue(j, arrayIdx, elementIdx + i); } } @@ -618,7 +520,7 @@ public class VectorTVList extends TVList { indices.get(arrayIdx)[elementIdx + i] = size; if (bitMaps != null) { for (int j = 0; j < bitMaps.length; j++) { - if (bitMaps[j] != null && bitMaps[j].get(idx + i)) { + if (bitMaps[j] != null && bitMaps[j].isMarked(idx + i)) { markNullValue(j, arrayIdx, elementIdx + i); } } @@ -673,6 +575,8 @@ public class VectorTVList extends TVList { bitMaps.add(null); } } + + // if the bitmap in columnIndex is null, init the bitmap of this column from the beginning if (bitMaps.get(columnIndex) == null) { List<BitMap> columnBitMaps = new ArrayList<>(); for (int i = 0; i < values.get(columnIndex).size(); i++) { @@ -680,6 +584,8 @@ public class VectorTVList extends TVList { } bitMaps.set(columnIndex, columnBitMaps); } + + // mark the null value in the current bitmap bitMaps.get(columnIndex).get(arrayIndex).mark(elementIndex); } diff --git a/session/src/main/java/org/apache/iotdb/session/Session.java b/session/src/main/java/org/apache/iotdb/session/Session.java index 95428f0..22c6add 100644 --- a/session/src/main/java/org/apache/iotdb/session/Session.java +++ b/session/src/main/java/org/apache/iotdb/session/Session.java @@ -1618,7 +1618,7 @@ public class Session { private BitMap sortBitMap(BitMap bitMap, Integer[] index) { BitMap sortedBitMap = new BitMap(bitMap.getSize()); for (int i = 0; i < index.length; i++) { - if (bitMap.get(index[i])) { + if (bitMap.isMarked(index[i])) { sortedBitMap.mark(i); } } diff --git a/session/src/main/java/org/apache/iotdb/session/SessionUtils.java b/session/src/main/java/org/apache/iotdb/session/SessionUtils.java index 4f98c63..5c799a4 100644 --- a/session/src/main/java/org/apache/iotdb/session/SessionUtils.java +++ b/session/src/main/java/org/apache/iotdb/session/SessionUtils.java @@ -42,17 +42,12 @@ public class SessionUtils { @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning public static ByteBuffer getValueBuffer(Tablet tablet) { - ByteBuffer valueBuffer = ByteBuffer.allocate(tablet.getValueBytesSize()); - boolean hasBitMaps = (tablet.bitMaps != null); + ByteBuffer valueBuffer = ByteBuffer.allocate(tablet.getTotalValueOccupation()); + boolean hasBitMaps = tablet.bitMaps != null; valueBuffer.put(BytesUtils.boolToByte(hasBitMaps)); if (hasBitMaps) { for (BitMap bitMap : tablet.bitMaps) { - boolean columnHasNull; - if (bitMap == null || bitMap.isAllZero()) { - columnHasNull = false; - } else { - columnHasNull = true; - } + boolean columnHasNull = bitMap != null && !bitMap.isAllUnmarked(); valueBuffer.put(BytesUtils.boolToByte(columnHasNull)); if (columnHasNull) { @@ -86,7 +81,7 @@ public class SessionUtils { for (int index = 0; index < tablet.rowSize; index++) { if (tablet.bitMaps == null || tablet.bitMaps[i] == null - || !tablet.bitMaps[i].get(index)) { + || !tablet.bitMaps[i].isMarked(index)) { valueBuffer.putInt(intValues[index]); } else { valueBuffer.putInt(Integer.MIN_VALUE); @@ -98,7 +93,7 @@ public class SessionUtils { for (int index = 0; index < tablet.rowSize; index++) { if (tablet.bitMaps == null || tablet.bitMaps[i] == null - || !tablet.bitMaps[i].get(index)) { + || !tablet.bitMaps[i].isMarked(index)) { valueBuffer.putLong(longValues[index]); } else { valueBuffer.putLong(Long.MIN_VALUE); @@ -110,7 +105,7 @@ public class SessionUtils { for (int index = 0; index < tablet.rowSize; index++) { if (tablet.bitMaps == null || tablet.bitMaps[i] == null - || !tablet.bitMaps[i].get(index)) { + || !tablet.bitMaps[i].isMarked(index)) { valueBuffer.putFloat(floatValues[index]); } else { valueBuffer.putFloat(Float.MIN_VALUE); @@ -122,7 +117,7 @@ public class SessionUtils { for (int index = 0; index < tablet.rowSize; index++) { if (tablet.bitMaps == null || tablet.bitMaps[i] == null - || !tablet.bitMaps[i].get(index)) { + || !tablet.bitMaps[i].isMarked(index)) { valueBuffer.putDouble(doubleValues[index]); } else { valueBuffer.putDouble(Double.MIN_VALUE); @@ -134,7 +129,7 @@ public class SessionUtils { for (int index = 0; index < tablet.rowSize; index++) { if (tablet.bitMaps == null || tablet.bitMaps[i] == null - || !tablet.bitMaps[i].get(index)) { + || !tablet.bitMaps[i].isMarked(index)) { valueBuffer.put(BytesUtils.boolToByte(boolValues[index])); } else { valueBuffer.put(BytesUtils.boolToByte(false)); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/Binary.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/Binary.java index 9d73967..fde1041 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/Binary.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/Binary.java @@ -30,6 +30,8 @@ import java.util.Arrays; public class Binary implements Comparable<Binary>, Serializable { private static final long serialVersionUID = 6394197743397020735L; + public static Binary EMPTY_VALUE = new Binary(""); + private byte[] values; /** if the bytes v is modified, the modification is visible to this binary. */ diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BitMap.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BitMap.java index bffbd4e..5177260 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BitMap.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BitMap.java @@ -57,7 +57,7 @@ public class BitMap { } /** returns the value of the bit with the specified index. */ - public boolean get(int position) { + public boolean isMarked(int position) { return (bits[position / Byte.SIZE] & BIT_UTIL[position % Byte.SIZE]) != 0; } @@ -75,8 +75,8 @@ public class BitMap { bits[position / Byte.SIZE] &= UNMARK_BIT_UTIL[position % Byte.SIZE]; } - /** whether all bits are zero */ - public boolean isAllZero() { + /** whether all bits are zero, i.e., no Null value */ + public boolean isAllUnmarked() { int j; for (j = 0; j < size / Byte.SIZE; j++) { if (bits[j] != (byte) 0) { @@ -91,8 +91,8 @@ public class BitMap { return true; } - /** whether all bits are one */ - public boolean isAllOne() { + /** whether all bits are one, i.e., all are Null */ + public boolean isAllMarked() { int j; for (j = 0; j < size / Byte.SIZE; j++) { if (bits[j] != (byte) 0XFF) { @@ -111,7 +111,7 @@ public class BitMap { public String toString() { StringBuffer res = new StringBuffer(); for (int i = 0; i < size; i++) { - res.append(get(i) ? 1 : 0); + res.append(isMarked(i) ? 1 : 0); } return res.toString(); } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/record/Tablet.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/record/Tablet.java index 3685448..bfd0c6f 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/record/Tablet.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/record/Tablet.java @@ -115,6 +115,7 @@ public class Tablet { timestamps[rowIndex] = timestamp; } + // (s1, s2) s3 public void addValue(String measurementId, int rowIndex, Object value) { int indexOfValue = measurementIndex.get(measurementId); IMeasurementSchema measurementSchema = schemas.get(indexOfValue); @@ -192,6 +193,8 @@ public class Tablet { private void createColumns() { // create timestamp column timestamps = new long[maxRowNumber]; + + // calculate total value column size int valueColumnsSize = 0; for (IMeasurementSchema schema : schemas) { if (schema instanceof VectorMeasurementSchema) { @@ -257,7 +260,7 @@ public class Tablet { } /** @return total bytes of values */ - public int getValueBytesSize() { + public int getTotalValueOccupation() { int valueOccupation = 0; // marker byte valueOccupation++; @@ -266,7 +269,7 @@ public class Tablet { for (BitMap bitMap : bitMaps) { // marker byte valueOccupation++; - if (bitMap != null && !bitMap.isAllZero()) { + if (bitMap != null && !bitMap.isAllUnmarked()) { valueOccupation += rowSize / Byte.SIZE + 1; } } @@ -274,18 +277,17 @@ public class Tablet { for (int i = 0; i < schemas.size(); i++) { IMeasurementSchema schema = schemas.get(i); if (schema instanceof MeasurementSchema) { - valueOccupation += calCalOccupation(schema.getType(), i); + valueOccupation += calOccupationOfOneColumn(schema.getType(), i); } else { for (TSDataType dataType : schema.getValueTSDataTypeList()) { - valueOccupation += calCalOccupation(dataType, i); + valueOccupation += calOccupationOfOneColumn(dataType, i); } } } return valueOccupation; } - /** total byte size that values occupies */ - private int calCalOccupation(TSDataType dataType, int i) { + private int calOccupationOfOneColumn(TSDataType dataType, int i) { int valueOccupation = 0; switch (dataType) { case BOOLEAN: diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/MeasurementSchema.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/MeasurementSchema.java index 584c105..4778ca6 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/MeasurementSchema.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/MeasurementSchema.java @@ -46,13 +46,6 @@ import java.util.Objects; public class MeasurementSchema implements IMeasurementSchema, Comparable<MeasurementSchema>, Serializable { - public static final MeasurementSchema TIME_SCHEMA = - new MeasurementSchema( - "time", - TSDataType.INT64, - TSEncoding.valueOf(TSFileDescriptor.getInstance().getConfig().getTimeEncoder()), - TSFileDescriptor.getInstance().getConfig().getCompressor()); - private String measurementId; private byte type; private byte encoding; diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/VectorMeasurementSchema.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/VectorMeasurementSchema.java index 98133d1..e6abdc6 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/VectorMeasurementSchema.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/VectorMeasurementSchema.java @@ -42,9 +42,10 @@ import java.util.Objects; public class VectorMeasurementSchema implements IMeasurementSchema, Comparable<VectorMeasurementSchema>, Serializable { - public static final String ALIGN_TIMESERIES_PREFIX = "$#$"; + public static final String VECTOR_NAME_PREFIX = "$#$"; - private String meausurementId; + // this is equal to the time id in this vector + private String vectorMeausurementId; private String[] measurements; private byte[] types; private byte[] encodings; @@ -59,7 +60,7 @@ public class VectorMeasurementSchema TSDataType[] types, TSEncoding[] encodings, CompressionType compressionType) { - this.meausurementId = measurementId; + this.vectorMeausurementId = measurementId; this.measurements = measurements; byte[] typesInByte = new byte[types.length]; for (int i = 0; i < types.length; i++) { @@ -87,19 +88,17 @@ public class VectorMeasurementSchema public VectorMeasurementSchema(String[] measurements, TSDataType[] types) { this.measurements = measurements; - byte[] typesInByte = new byte[types.length]; + this.types = new byte[types.length]; for (int i = 0; i < types.length; i++) { - typesInByte[i] = types[i].serialize(); + this.types[i] = types[i].serialize(); } - this.types = typesInByte; - byte[] encodings = new byte[types.length]; + this.encodings = new byte[types.length]; for (int i = 0; i < types.length; i++) { - encodings[i] = + this.encodings[i] = TSEncoding.valueOf(TSFileDescriptor.getInstance().getConfig().getValueEncoder()) .serialize(); } - this.encodings = encodings; this.encodingConverters = new TSEncodingBuilder[measurements.length]; this.compressor = TSFileDescriptor.getInstance().getConfig().getCompressor().serialize(); } @@ -116,7 +115,7 @@ public class VectorMeasurementSchema @Override public String getMeasurementId() { - return meausurementId; + return vectorMeausurementId; } @Override @@ -211,7 +210,7 @@ public class VectorMeasurementSchema public int serializeTo(ByteBuffer buffer) { int byteLen = 0; byteLen += - ReadWriteIOUtils.write(meausurementId.substring(ALIGN_TIMESERIES_PREFIX.length()), buffer); + ReadWriteIOUtils.write(vectorMeausurementId.substring(VECTOR_NAME_PREFIX.length()), buffer); byteLen += ReadWriteIOUtils.write(measurements.length, buffer); for (String measurementId : measurements) { @@ -233,7 +232,7 @@ public class VectorMeasurementSchema int byteLen = 0; byteLen += ReadWriteIOUtils.write( - meausurementId.substring(ALIGN_TIMESERIES_PREFIX.length()), outputStream); + vectorMeausurementId.substring(VECTOR_NAME_PREFIX.length()), outputStream); byteLen += ReadWriteIOUtils.write(measurements.length, outputStream); for (String measurementId : measurements) { @@ -269,8 +268,8 @@ public class VectorMeasurementSchema public static VectorMeasurementSchema deserializeFrom(InputStream inputStream) throws IOException { VectorMeasurementSchema vectorMeasurementSchema = new VectorMeasurementSchema(); - vectorMeasurementSchema.meausurementId = - ALIGN_TIMESERIES_PREFIX + ReadWriteIOUtils.readString(inputStream); + vectorMeasurementSchema.vectorMeausurementId = + VECTOR_NAME_PREFIX + ReadWriteIOUtils.readString(inputStream); int measurementSize = ReadWriteIOUtils.readInt(inputStream); String[] measurements = new String[measurementSize]; @@ -297,8 +296,8 @@ public class VectorMeasurementSchema public static VectorMeasurementSchema deserializeFrom(ByteBuffer buffer) { VectorMeasurementSchema vectorMeasurementSchema = new VectorMeasurementSchema(); - vectorMeasurementSchema.meausurementId = - ALIGN_TIMESERIES_PREFIX + ReadWriteIOUtils.readString(buffer); + vectorMeasurementSchema.vectorMeausurementId = + VECTOR_NAME_PREFIX + ReadWriteIOUtils.readString(buffer); int measurementSize = ReadWriteIOUtils.readInt(buffer); String[] measurements = new String[measurementSize]; for (int i = 0; i < measurementSize; i++) { diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/BitMapTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/BitMapTest.java index a17678a..ef63b8a 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/BitMapTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/BitMapTest.java @@ -30,24 +30,24 @@ public class BitMapTest { public void testMarkAndUnMark() { BitMap bitmap = new BitMap(100); assertEquals(100, bitmap.getSize()); - assertTrue(bitmap.isAllZero()); - assertFalse(bitmap.isAllOne()); + assertTrue(bitmap.isAllUnmarked()); + assertFalse(bitmap.isAllMarked()); for (int i = 0; i < 100; i++) { bitmap.mark(i); - assertTrue(bitmap.get(i)); + assertTrue(bitmap.isMarked(i)); if (i == 50) { - assertFalse(bitmap.isAllOne()); - assertFalse(bitmap.isAllZero()); + assertFalse(bitmap.isAllMarked()); + assertFalse(bitmap.isAllUnmarked()); } } - assertTrue(bitmap.isAllOne()); - assertFalse(bitmap.isAllZero()); + assertTrue(bitmap.isAllMarked()); + assertFalse(bitmap.isAllUnmarked()); for (int i = 0; i < 100; i++) { bitmap.unmark(i); - assertFalse(bitmap.get(i)); + assertFalse(bitmap.isMarked(i)); } - assertTrue(bitmap.isAllZero()); - assertFalse(bitmap.isAllOne()); + assertTrue(bitmap.isAllUnmarked()); + assertFalse(bitmap.isAllMarked()); } @Test @@ -61,7 +61,7 @@ public class BitMapTest { BitMap bitmap2 = new BitMap(bitmap1.getSize(), bitmap1.getByteArray()); assertEquals(100, bitmap2.getSize()); for (int i = 0; i < 100; i++) { - assertEquals(bitmap1.get(i), bitmap2.get(i)); + assertEquals(bitmap1.isMarked(i), bitmap2.isMarked(i)); } } }
