This is an automated email from the ASF dual-hosted git repository. leirui pushed a commit to branch research/M4-visualization in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit fcd4555e50609b9f318966ef1313fa6e8f0d8b1b Author: Lei Rui <[email protected]> AuthorDate: Fri Jul 7 11:52:04 2023 +0800 add log --- .../dataset/groupby/LocalGroupByExecutor4CPV.java | 20 +++++++----- .../tsfile/file/metadata/TimeseriesMetadata.java | 3 +- .../file/metadata/statistics/Statistics.java | 36 ++++++++++++++++++---- .../file/metadata/statistics/ValueIndex.java | 3 ++ 4 files changed, 47 insertions(+), 15 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java index 294129523e3..f30f936663f 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java +++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java @@ -153,7 +153,7 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor { int BP_v_int = ((IntegerStatistics) statistics).getMinValue(); int TP_v_int = ((IntegerStatistics) statistics).getMaxValue(); M4_CHUNK_METADATA.debug( - "M4_CHUNK_METADATA,{},{},{},{},{},{},{},{},{},{}", + "M4_CHUNK_METADATA,{},{},{},{},{},{},{},{},{},{},{}", FP_t, LP_t, BP_t, @@ -163,7 +163,8 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor { BP_v_int, TP_v_int, chunkSuit4CPV.getChunkMetadata().getVersion(), - chunkSuit4CPV.getChunkMetadata().getOffsetOfChunkHeader()); + chunkSuit4CPV.getChunkMetadata().getOffsetOfChunkHeader(), + statistics.getCount()); break; case INT64: long FP_v_long = ((LongStatistics) statistics).getFirstValue(); @@ -171,7 +172,7 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor { long BP_v_long = ((LongStatistics) statistics).getMinValue(); long TP_v_long = ((LongStatistics) statistics).getMaxValue(); M4_CHUNK_METADATA.debug( - "M4_CHUNK_METADATA,{},{},{},{},{},{},{},{},{},{}", + "M4_CHUNK_METADATA,{},{},{},{},{},{},{},{},{},{},{}", FP_t, LP_t, BP_t, @@ -181,7 +182,8 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor { BP_v_long, TP_v_long, chunkSuit4CPV.getChunkMetadata().getVersion(), - chunkSuit4CPV.getChunkMetadata().getOffsetOfChunkHeader()); + chunkSuit4CPV.getChunkMetadata().getOffsetOfChunkHeader(), + statistics.getCount()); break; case FLOAT: float FP_v_float = ((FloatStatistics) statistics).getFirstValue(); @@ -189,7 +191,7 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor { float BP_v_float = ((FloatStatistics) statistics).getMinValue(); float TP_v_float = ((FloatStatistics) statistics).getMaxValue(); M4_CHUNK_METADATA.debug( - "M4_CHUNK_METADATA,{},{},{},{},{},{},{},{},{},{}", + "M4_CHUNK_METADATA,{},{},{},{},{},{},{},{},{},{},{}", FP_t, LP_t, BP_t, @@ -199,7 +201,8 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor { BP_v_float, TP_v_float, chunkSuit4CPV.getChunkMetadata().getVersion(), - chunkSuit4CPV.getChunkMetadata().getOffsetOfChunkHeader()); + chunkSuit4CPV.getChunkMetadata().getOffsetOfChunkHeader(), + statistics.getCount()); break; case DOUBLE: double FP_v_double = ((DoubleStatistics) statistics).getFirstValue(); @@ -207,7 +210,7 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor { double BP_v_double = ((DoubleStatistics) statistics).getMinValue(); double TP_v_double = ((DoubleStatistics) statistics).getMaxValue(); M4_CHUNK_METADATA.debug( - "M4_CHUNK_METADATA,{},{},{},{},{},{},{},{},{},{}", + "M4_CHUNK_METADATA,{},{},{},{},{},{},{},{},{},{},{}", FP_t, LP_t, BP_t, @@ -217,7 +220,8 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor { BP_v_double, TP_v_double, chunkSuit4CPV.getChunkMetadata().getVersion(), - chunkSuit4CPV.getChunkMetadata().getOffsetOfChunkHeader()); + chunkSuit4CPV.getChunkMetadata().getOffsetOfChunkHeader(), + statistics.getCount()); break; default: throw new QueryProcessException("unsupported data type!"); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java index f8bea3156b4..e2301e15b95 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java @@ -131,7 +131,8 @@ public class TimeseriesMetadata implements Accountable { byteLen += ReadWriteIOUtils.write(dataType, outputStream); byteLen += ReadWriteForEncodingUtils.writeUnsignedVarInt(chunkMetaDataListDataSize, outputStream); - byteLen += statistics.serialize(outputStream); + // log false to avoid interfere with chunk statistics log info + byteLen += statistics.serialize(outputStream, false); chunkMetadataListBuffer.writeTo(outputStream); byteLen += chunkMetadataListBuffer.size(); return byteLen; diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java index 804a17d53d4..aa1d544205f 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java @@ -135,13 +135,27 @@ public abstract class Statistics<T> { // value statistics of different data type byteLen += serializeStats(outputStream); // serialize stepRegress - byteLen += serializeStepRegress(outputStream); + byteLen += serializeStepRegress(outputStream, true); // serialize value index - byteLen += serializeValueIndex(outputStream); + byteLen += serializeValueIndex(outputStream, true); return byteLen; } - int serializeValueIndex(OutputStream outputStream) throws IOException { + public int serialize(OutputStream outputStream, boolean log) throws IOException { + int byteLen = 0; + byteLen += ReadWriteForEncodingUtils.writeUnsignedVarInt(count, outputStream); + byteLen += ReadWriteIOUtils.write(startTime, outputStream); + byteLen += ReadWriteIOUtils.write(endTime, outputStream); + // value statistics of different data type + byteLen += serializeStats(outputStream); + // serialize stepRegress + byteLen += serializeStepRegress(outputStream, log); + // serialize value index + byteLen += serializeValueIndex(outputStream, log); + return byteLen; + } + + int serializeValueIndex(OutputStream outputStream, boolean log) throws IOException { valueIndex.learn(); // ensure executed once and only once int byteLen = 0; byteLen += ReadWriteIOUtils.write(valueIndex.idxOut.size(), outputStream); @@ -155,7 +169,12 @@ public abstract class Statistics<T> { byteLen += valueIndex.valueOut.size(); byteLen += ReadWriteIOUtils.write(valueIndex.errorBound, outputStream); - LOG.info("value_index_serialize_byteLen,{}", byteLen); + if (log) { + LOG.info( + "value_index_serialize_byteLen,{},modelPoint_count_exceptFPvLPv,{}", + byteLen, + valueIndex.modelPointCount); + } return byteLen; } @@ -163,7 +182,7 @@ public abstract class Statistics<T> { * slope, m: the number of segment keys, m-2 segment keys in between when m>=2. The first and the * last segment keys are not serialized here, because they are minTime and endTime respectively. */ - int serializeStepRegress(OutputStream outputStream) throws IOException { + int serializeStepRegress(OutputStream outputStream, boolean log) throws IOException { stepRegress.learn(); // ensure executed once and only once int byteLen = 0; byteLen += ReadWriteIOUtils.write(stepRegress.getSlope(), outputStream); // K @@ -173,7 +192,12 @@ public abstract class Statistics<T> { for (int i = 1; i < segmentKeys.size() - 1; i++) { // t2,t3,...,tm-1 byteLen += ReadWriteIOUtils.write(segmentKeys.get(i), outputStream); } - LOG.info("time_index_serialize_byteLen,{}", byteLen); + if (log) { + LOG.info( + "time_index_serialize_byteLen,{},segmentKeys_size_includeFPtLPt,{}", + byteLen, + segmentKeys.size()); + } return byteLen; } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/ValueIndex.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/ValueIndex.java index 6cefa6088ec..bd16cdfc074 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/ValueIndex.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/ValueIndex.java @@ -31,6 +31,8 @@ public class ValueIndex { // causing learn() executed more than once!! private boolean isLearned = false; + public int modelPointCount = 0; // except the first and last points + private double stdDev = 0; // standard deviation of intervals private long count = 0; private double sumX2 = 0.0; @@ -92,6 +94,7 @@ public class ValueIndex { // the first point value is stored as FirstValue in statistics, so here no need store the // first point // the last point won't be checked by the if SDT encode logic + modelPointCount++; idxEncoder.encode((int) sdtEncoder.getTime(), idxOut); valueEncoder.encode(sdtEncoder.getDoubleValue(), valueOut); if (!hasDataToFlush) {
