This is an automated email from the ASF dual-hosted git repository. leirui pushed a commit to branch research/M4-visualization in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit da7fa9210a0b3847a47d589fbfec453ebcf505ab Author: Lei Rui <[email protected]> AuthorDate: Tue Jul 4 18:42:19 2023 +0800 save first and last points --- .../file/metadata/statistics/Statistics.java | 98 +++++++++++++++++----- .../file/metadata/statistics/ValueIndex.java | 37 +++++--- 2 files changed, 101 insertions(+), 34 deletions(-) diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java index 18e50e7be27..c712d1967f0 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java @@ -130,17 +130,17 @@ public abstract class Statistics<T> { byteLen += ReadWriteForEncodingUtils.writeUnsignedVarInt(count, outputStream); byteLen += ReadWriteIOUtils.write(startTime, outputStream); byteLen += ReadWriteIOUtils.write(endTime, outputStream); + // value statistics of different data type + byteLen += serializeStats(outputStream); // serialize stepRegress byteLen += serializeStepRegress(outputStream); - // TODO serialize value index + // serialize value index byteLen += serializeValueIndex(outputStream); - // value statistics of different data type - byteLen += serializeStats(outputStream); return byteLen; } int serializeValueIndex(OutputStream outputStream) throws IOException { - valueIndex.learn(); // TODO ensure executed once and only once + valueIndex.learn(); // ensure executed once and only once int byteLen = 0; byteLen += ReadWriteIOUtils.write(valueIndex.idxOut.size(), outputStream); outputStream.write( @@ -162,7 +162,7 @@ public abstract class Statistics<T> { * last segment keys are not serialized here, because they are minTime and endTime respectively. */ int serializeStepRegress(OutputStream outputStream) throws IOException { - stepRegress.learn(); // TODO ensure executed once and only once + stepRegress.learn(); // ensure executed once and only once int byteLen = 0; byteLen += ReadWriteIOUtils.write(stepRegress.getSlope(), outputStream); // K DoubleArrayList segmentKeys = stepRegress.getSegmentKeys(); @@ -574,35 +574,91 @@ public abstract class Statistics<T> { statistics.setCount(ReadWriteForEncodingUtils.readUnsignedVarInt(buffer)); statistics.setStartTime(ReadWriteIOUtils.readLong(buffer)); statistics.setEndTime(ReadWriteIOUtils.readLong(buffer)); - statistics.deserializeStepRegress(buffer); - statistics.deserializeValueIndex(buffer); // TODO statistics.deserialize(buffer); + statistics.deserializeStepRegress(buffer); + statistics.deserializeValueIndex(buffer); statistics.isEmpty = false; return statistics; } void deserializeValueIndex(ByteBuffer buffer) throws IOException { + // add the first point + valueIndex.modelPointIdx_list.add(1); + switch (getType()) { + case INT32: + int intV = (int) getFirstValue(); + valueIndex.modelPointVal_list.add((double) intV); + break; + case INT64: + long longV = (long) getFirstValue(); + valueIndex.modelPointVal_list.add((double) longV); + break; + case FLOAT: + float floatV = (float) getFirstValue(); + valueIndex.modelPointVal_list.add((double) floatV); + break; + case DOUBLE: + double doubleV = (double) getFirstValue(); + valueIndex.modelPointVal_list.add(doubleV); + break; + default: + throw new IOException("unsupported"); + } + int idxSize = ReadWriteIOUtils.readInt(buffer); - ByteBuffer idxBuffer = buffer.slice(); - idxBuffer.limit(idxSize); + if (idxSize > 0) { + ByteBuffer idxBuffer = buffer.slice(); + idxBuffer.limit(idxSize); + Decoder idxDecoder = new IntDeltaDecoder(); + while (idxDecoder.hasNext(idxBuffer)) { + int idx = idxDecoder.readInt(idxBuffer); + valueIndex.modelPointIdx_list.add(idx); + } + } + buffer.position(buffer.position() + idxSize); - Decoder idxDecoder = new IntDeltaDecoder(); - while (idxDecoder.hasNext(idxBuffer)) { - int idx = idxDecoder.readInt(idxBuffer); - valueIndex.modelPointIdx_list.add(idx); + int valueSize = ReadWriteIOUtils.readInt(buffer); + if (valueSize > 0) { + ByteBuffer valueBuffer = buffer.slice(); + valueBuffer.limit(valueSize); + Decoder valueDecoder = new DoublePrecisionDecoderV2(); + while (valueDecoder.hasNext(valueBuffer)) { + double value = valueDecoder.readDouble(valueBuffer); + valueIndex.modelPointVal_list.add(value); + } } - int valueSize = ReadWriteIOUtils.readInt(buffer); // valueOut.size() - ByteBuffer valueBuffer = buffer.slice(); - valueBuffer.limit(valueSize); - buffer.position(buffer.position() + valueSize); - Decoder valueDecoder = new DoublePrecisionDecoderV2(); - while (valueDecoder.hasNext(valueBuffer)) { - double value = valueDecoder.readDouble(valueBuffer); - valueIndex.modelPointVal_list.add(value); + // add the last point except the first point + if (count >= 2) { // otherwise only one point no need to store again + valueIndex.modelPointIdx_list.add(count); + switch (getType()) { + case INT32: + int intV = (int) getLastValue(); + valueIndex.modelPointVal_list.add((double) intV); + break; + case INT64: + long longV = (long) getLastValue(); + valueIndex.modelPointVal_list.add((double) longV); + break; + case FLOAT: + float floatV = (float) getLastValue(); + valueIndex.modelPointVal_list.add((double) floatV); + break; + case DOUBLE: + double doubleV = (double) getLastValue(); + valueIndex.modelPointVal_list.add(doubleV); + break; + default: + throw new IOException("unsupported"); + } } + // error bound + buffer.position(buffer.position() + valueSize); valueIndex.errorBound = ReadWriteIOUtils.readDouble(buffer); + + // System.out.println(valueIndex.modelPointIdx_list); + // System.out.println(valueIndex.modelPointVal_list); } void deserializeStepRegress(ByteBuffer byteBuffer) { diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/ValueIndex.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/ValueIndex.java index 24deea0a871..1ad935e6ef8 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/ValueIndex.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/ValueIndex.java @@ -17,11 +17,6 @@ public class ValueIndex { public DoublePrecisionEncoderV2 valueEncoder = new DoublePrecisionEncoderV2(); public PublicBAOS idxOut = new PublicBAOS(); public PublicBAOS valueOut = new PublicBAOS(); - public int lastIdx = 0; - public int lastIntValue = 0; - public long lastLongValue = 0; - public float lastFloatValue = 0; - public double lastDoubleValue = 0; public IntArrayList modelPointIdx_list = new IntArrayList(); public DoubleArrayList modelPointVal_list = new DoubleArrayList(); @@ -82,19 +77,35 @@ public class ValueIndex { isLearned = true; initForLearn(); // set self-adapting CompDeviation for sdtEncoder int pos = 0; + boolean hasDataToFlush = false; for (double v : values.toArray()) { pos++; // starting from 1 if (sdtEncoder.encodeDouble(pos, v)) { - idxEncoder.encode((int) sdtEncoder.getTime(), idxOut); - valueEncoder.encode(sdtEncoder.getDoubleValue(), valueOut); + if (pos > 1) { + // the first point value is stored as FirstValue in statistics, so here no need store the + // first point + // the last point won't be checked by the if SDT encode logic + idxEncoder.encode((int) sdtEncoder.getTime(), idxOut); + valueEncoder.encode(sdtEncoder.getDoubleValue(), valueOut); + if (!hasDataToFlush) { + hasDataToFlush = true; + } + } } } - // add the last point except the first point - if (values.size() >= 2) { // means there is last point except the first point - idxEncoder.encode(pos, idxOut); - valueEncoder.encode(values.getLast(), valueOut); + // // add the last point except the first point + // if (values.size() >= 2) { // means there is last point except the first point + // idxEncoder.encode(pos, idxOut); + // valueEncoder.encode(values.getLast(), valueOut); + // } + + if (hasDataToFlush) { + // otherwise no need flush, because GorillaV2 encoding will output NaN even if + // hasDataToFlush=false + idxEncoder.flush(idxOut); // necessary + valueEncoder.flush(valueOut); // necessary } - idxEncoder.flush(idxOut); - valueEncoder.flush(valueOut); + + values = null; // raw values are not needed any more } }
