This is an automated email from the ASF dual-hosted git repository. leirui pushed a commit to branch add_m4 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 34e4f28122882d607938241672b98bd016bdc1ec Author: Lei Rui <[email protected]> AuthorDate: Sat Mar 22 23:37:58 2025 +0800 add --- .../resources/conf/iotdb-engine.properties | 2 + .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 7 ++ .../iotdb/tsfile/common/conf/TSFileConfig.java | 10 +++ .../iotdb/tsfile/common/conf/TSFileDescriptor.java | 3 + .../file/metadata/statistics/DoubleStatistics.java | 19 ++++-- .../file/metadata/statistics/FloatStatistics.java | 19 ++++-- .../metadata/statistics/IntegerStatistics.java | 19 ++++-- .../file/metadata/statistics/LongStatistics.java | 19 ++++-- .../file/metadata/statistics/Statistics.java | 76 ++++++++++++++-------- 9 files changed, 124 insertions(+), 50 deletions(-) diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties index 95228d09694..0e095af770b 100644 --- a/server/src/assembly/resources/conf/iotdb-engine.properties +++ b/server/src/assembly/resources/conf/iotdb-engine.properties @@ -613,6 +613,8 @@ use_TimeIndex=true use_Mad=false +write_m4_lsm=true + use_ValueIndex=true error_Param=1 diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 4711470e002..bcd7bb971f3 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -927,6 +927,13 @@ public class IoTDBDescriptor { properties.getProperty( "use_Mad", Boolean.toString(TSFileDescriptor.getInstance().getConfig().isUseMad())))); + TSFileDescriptor.getInstance() + .getConfig() + .setWriteM4LSM( + Boolean.parseBoolean( + properties.getProperty( + "write_m4_lsm", + Boolean.toString(TSFileDescriptor.getInstance().getConfig().isWriteM4LSM())))); TSFileDescriptor.getInstance() .getConfig() .setErrorParam( diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java index c745982c5a4..436d01697b2 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java @@ -37,6 +37,8 @@ public class TSFileConfig implements Serializable { private boolean useMad = false; + private boolean writeM4LSM = true; + private int errorParam = 1; /** encoding configuration */ @@ -191,6 +193,14 @@ public class TSFileConfig implements Serializable { this.useMad = useMad; } + public boolean isWriteM4LSM() { + return writeM4LSM; + } + + public void setWriteM4LSM(boolean writeM4LSM) { + this.writeM4LSM = writeM4LSM; + } + public int getErrorParam() { return errorParam; } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java index 3aa322b4e89..44cb1f9919f 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java @@ -118,6 +118,9 @@ public class TSFileDescriptor { conf.setUseMad( Boolean.parseBoolean( properties.getProperty("use_Mad", Boolean.toString(conf.isUseMad())))); + conf.setWriteM4LSM( + Boolean.parseBoolean( + properties.getProperty("write_m4_lsm", Boolean.toString(conf.isWriteM4LSM())))); conf.setErrorParam( Integer.parseInt( properties.getProperty("error_Param", Integer.toString(conf.getErrorParam())))); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatistics.java index 8771ff089dd..d7ec42a53b5 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatistics.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatistics.java @@ -18,6 +18,7 @@ */ package org.apache.iotdb.tsfile.file.metadata.statistics; +import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; import org.apache.iotdb.tsfile.exception.filter.StatisticsClassException; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; @@ -128,8 +129,10 @@ public class DoubleStatistics extends Statistics<Double> { long topTimestamp, double lastValue, double sumValue) { - updateMinInfo(minValue, bottomTimestamp); - updateMaxInfo(maxValue, topTimestamp); + if (TSFileDescriptor.getInstance().getConfig().isWriteM4LSM()) { + updateMinInfo(minValue, bottomTimestamp); + updateMaxInfo(maxValue, topTimestamp); + } this.sumValue += sumValue; this.lastValue = lastValue; } @@ -287,8 +290,10 @@ public class DoubleStatistics extends Statistics<Double> { @Override public int serializeStats(OutputStream outputStream) throws IOException { int byteLen = 0; - byteLen += ReadWriteIOUtils.write(minInfo, minMaxDataType, outputStream); - byteLen += ReadWriteIOUtils.write(maxInfo, minMaxDataType, outputStream); + if (TSFileDescriptor.getInstance().getConfig().isWriteM4LSM()) { + byteLen += ReadWriteIOUtils.write(minInfo, minMaxDataType, outputStream); + byteLen += ReadWriteIOUtils.write(maxInfo, minMaxDataType, outputStream); + } byteLen += ReadWriteIOUtils.write(firstValue, outputStream); byteLen += ReadWriteIOUtils.write(lastValue, outputStream); byteLen += ReadWriteIOUtils.write(sumValue, outputStream); @@ -308,8 +313,10 @@ public class DoubleStatistics extends Statistics<Double> { /** @author Yuyuan Kang */ @Override public void deserialize(ByteBuffer byteBuffer) { - this.minInfo = ReadWriteIOUtils.readMinMaxInfo(byteBuffer, minMaxDataType); - this.maxInfo = ReadWriteIOUtils.readMinMaxInfo(byteBuffer, minMaxDataType); + if (TSFileDescriptor.getInstance().getConfig().isWriteM4LSM()) { + this.minInfo = ReadWriteIOUtils.readMinMaxInfo(byteBuffer, minMaxDataType); + this.maxInfo = ReadWriteIOUtils.readMinMaxInfo(byteBuffer, minMaxDataType); + } this.firstValue = ReadWriteIOUtils.readDouble(byteBuffer); this.lastValue = ReadWriteIOUtils.readDouble(byteBuffer); this.sumValue = ReadWriteIOUtils.readDouble(byteBuffer); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatistics.java index c79aafb0a9f..3ab6e53d700 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatistics.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatistics.java @@ -18,6 +18,7 @@ */ package org.apache.iotdb.tsfile.file.metadata.statistics; +import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; import org.apache.iotdb.tsfile.exception.filter.StatisticsClassException; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; @@ -130,8 +131,10 @@ public class FloatStatistics extends Statistics<Float> { long topTimestamp, float last, double sumValue) { - updateMinInfo(minValue, bottomTimestamp); - updateMaxInfo(maxValue, topTimestamp); + if (TSFileDescriptor.getInstance().getConfig().isWriteM4LSM()) { + updateMinInfo(minValue, bottomTimestamp); + updateMaxInfo(maxValue, topTimestamp); + } this.sumValue += sumValue; this.lastValue = last; } @@ -289,8 +292,10 @@ public class FloatStatistics extends Statistics<Float> { @Override public int serializeStats(OutputStream outputStream) throws IOException { int byteLen = 0; - byteLen += ReadWriteIOUtils.write(minInfo, minMaxDataType, outputStream); - byteLen += ReadWriteIOUtils.write(maxInfo, minMaxDataType, outputStream); + if (TSFileDescriptor.getInstance().getConfig().isWriteM4LSM()) { + byteLen += ReadWriteIOUtils.write(minInfo, minMaxDataType, outputStream); + byteLen += ReadWriteIOUtils.write(maxInfo, minMaxDataType, outputStream); + } byteLen += ReadWriteIOUtils.write(firstValue, outputStream); byteLen += ReadWriteIOUtils.write(lastValue, outputStream); byteLen += ReadWriteIOUtils.write(sumValue, outputStream); @@ -310,8 +315,10 @@ public class FloatStatistics extends Statistics<Float> { /** @author Yuyuan Kang */ @Override public void deserialize(ByteBuffer byteBuffer) { - this.minInfo = ReadWriteIOUtils.readMinMaxInfo(byteBuffer, minMaxDataType); - this.maxInfo = ReadWriteIOUtils.readMinMaxInfo(byteBuffer, minMaxDataType); + if (TSFileDescriptor.getInstance().getConfig().isWriteM4LSM()) { + this.minInfo = ReadWriteIOUtils.readMinMaxInfo(byteBuffer, minMaxDataType); + this.maxInfo = ReadWriteIOUtils.readMinMaxInfo(byteBuffer, minMaxDataType); + } this.firstValue = ReadWriteIOUtils.readFloat(byteBuffer); this.lastValue = ReadWriteIOUtils.readFloat(byteBuffer); this.sumValue = ReadWriteIOUtils.readDouble(byteBuffer); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/IntegerStatistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/IntegerStatistics.java index 2bb2052ee30..2d2f4809c18 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/IntegerStatistics.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/IntegerStatistics.java @@ -18,6 +18,7 @@ */ package org.apache.iotdb.tsfile.file.metadata.statistics; +import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; import org.apache.iotdb.tsfile.exception.filter.StatisticsClassException; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; @@ -105,8 +106,10 @@ public class IntegerStatistics extends Statistics<Integer> { long topTimestamp, int lastValue, long sumValue) { - updateMinInfo(minValue, bottomTimestamp); - updateMaxInfo(maxValue, topTimestamp); + if (TSFileDescriptor.getInstance().getConfig().isWriteM4LSM()) { + updateMinInfo(minValue, bottomTimestamp); + updateMaxInfo(maxValue, topTimestamp); + } this.sumValue += sumValue; this.lastValue = lastValue; } @@ -290,8 +293,10 @@ public class IntegerStatistics extends Statistics<Integer> { @Override public int serializeStats(OutputStream outputStream) throws IOException { int byteLen = 0; - byteLen += ReadWriteIOUtils.write(minInfo, minMaxDataType, outputStream); - byteLen += ReadWriteIOUtils.write(maxInfo, minMaxDataType, outputStream); + if (TSFileDescriptor.getInstance().getConfig().isWriteM4LSM()) { + byteLen += ReadWriteIOUtils.write(minInfo, minMaxDataType, outputStream); + byteLen += ReadWriteIOUtils.write(maxInfo, minMaxDataType, outputStream); + } byteLen += ReadWriteIOUtils.write(firstValue, outputStream); byteLen += ReadWriteIOUtils.write(lastValue, outputStream); byteLen += ReadWriteIOUtils.write(sumValue, outputStream); @@ -311,8 +316,10 @@ public class IntegerStatistics extends Statistics<Integer> { /** @author Yuyuan Kang */ @Override public void deserialize(ByteBuffer byteBuffer) { - this.minInfo = ReadWriteIOUtils.readMinMaxInfo(byteBuffer, minMaxDataType); - this.maxInfo = ReadWriteIOUtils.readMinMaxInfo(byteBuffer, minMaxDataType); + if (TSFileDescriptor.getInstance().getConfig().isWriteM4LSM()) { + this.minInfo = ReadWriteIOUtils.readMinMaxInfo(byteBuffer, minMaxDataType); + this.maxInfo = ReadWriteIOUtils.readMinMaxInfo(byteBuffer, minMaxDataType); + } this.firstValue = ReadWriteIOUtils.readInt(byteBuffer); this.lastValue = ReadWriteIOUtils.readInt(byteBuffer); this.sumValue = ReadWriteIOUtils.readLong(byteBuffer); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatistics.java index 2ab67480fb5..04bcf1577de 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatistics.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatistics.java @@ -18,6 +18,7 @@ */ package org.apache.iotdb.tsfile.file.metadata.statistics; +import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; import org.apache.iotdb.tsfile.exception.filter.StatisticsClassException; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; @@ -99,8 +100,10 @@ public class LongStatistics extends Statistics<Long> { long topTimestamp, long lastValue, double sumValue) { - updateMinInfo(minValue, bottomTimestamp); - updateMaxInfo(maxValue, topTimestamp); + if (TSFileDescriptor.getInstance().getConfig().isWriteM4LSM()) { + updateMinInfo(minValue, bottomTimestamp); + updateMaxInfo(maxValue, topTimestamp); + } this.sumValue += sumValue; this.lastValue = lastValue; } @@ -291,8 +294,10 @@ public class LongStatistics extends Statistics<Long> { @Override public int serializeStats(OutputStream outputStream) throws IOException { int byteLen = 0; - byteLen += ReadWriteIOUtils.write(minInfo, minMaxDataType, outputStream); - byteLen += ReadWriteIOUtils.write(maxInfo, minMaxDataType, outputStream); + if (TSFileDescriptor.getInstance().getConfig().isWriteM4LSM()) { + byteLen += ReadWriteIOUtils.write(minInfo, minMaxDataType, outputStream); + byteLen += ReadWriteIOUtils.write(maxInfo, minMaxDataType, outputStream); + } byteLen += ReadWriteIOUtils.write(firstValue, outputStream); byteLen += ReadWriteIOUtils.write(lastValue, outputStream); byteLen += ReadWriteIOUtils.write(sumValue, outputStream); @@ -312,8 +317,10 @@ public class LongStatistics extends Statistics<Long> { /** @author Yuyuan Kang */ @Override public void deserialize(ByteBuffer byteBuffer) { - this.minInfo = ReadWriteIOUtils.readMinMaxInfo(byteBuffer, minMaxDataType); - this.maxInfo = ReadWriteIOUtils.readMinMaxInfo(byteBuffer, minMaxDataType); + if (TSFileDescriptor.getInstance().getConfig().isWriteM4LSM()) { + this.minInfo = ReadWriteIOUtils.readMinMaxInfo(byteBuffer, minMaxDataType); + this.maxInfo = ReadWriteIOUtils.readMinMaxInfo(byteBuffer, minMaxDataType); + } this.firstValue = ReadWriteIOUtils.readLong(byteBuffer); this.lastValue = ReadWriteIOUtils.readLong(byteBuffer); this.sumValue = ReadWriteIOUtils.readDouble(byteBuffer); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java index da2b27e9233..22a70b62984 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java @@ -18,6 +18,7 @@ */ package org.apache.iotdb.tsfile.file.metadata.statistics; +import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; import org.apache.iotdb.tsfile.encoding.decoder.Decoder; import org.apache.iotdb.tsfile.encoding.decoder.PlainDecoder; import org.apache.iotdb.tsfile.exception.filter.StatisticsClassException; @@ -133,10 +134,13 @@ public abstract class Statistics<T> { byteLen += ReadWriteIOUtils.write(endTime, outputStream); // value statistics of different data type byteLen += serializeStats(outputStream); - // serialize stepRegress - byteLen += serializeStepRegress(outputStream, true); - // serialize value index - byteLen += serializeValueIndex(outputStream, true); + + if (TSFileDescriptor.getInstance().getConfig().isWriteM4LSM()) { + // serialize stepRegress + byteLen += serializeStepRegress(outputStream, true); + // serialize value index + byteLen += serializeValueIndex(outputStream, true); + } return byteLen; } @@ -147,10 +151,12 @@ public abstract class Statistics<T> { byteLen += ReadWriteIOUtils.write(endTime, outputStream); // value statistics of different data type byteLen += serializeStats(outputStream); - // serialize stepRegress - byteLen += serializeStepRegress(outputStream, log); - // serialize value index - byteLen += serializeValueIndex(outputStream, log); + if (TSFileDescriptor.getInstance().getConfig().isWriteM4LSM()) { + // serialize stepRegress + byteLen += serializeStepRegress(outputStream, log); + // serialize value index + byteLen += serializeValueIndex(outputStream, log); + } return byteLen; } @@ -312,8 +318,10 @@ public abstract class Statistics<T> { endTime = time; } // update time index - updateStepRegress(time); - updateValueIndex(value); + if (TSFileDescriptor.getInstance().getConfig().isWriteM4LSM()) { + updateStepRegress(time); + updateValueIndex(value); + } updateStats(value, time); } @@ -326,8 +334,10 @@ public abstract class Statistics<T> { if (time > this.endTime) { endTime = time; } - updateStepRegress(time); - updateValueIndex(value); + if (TSFileDescriptor.getInstance().getConfig().isWriteM4LSM()) { + updateStepRegress(time); + updateValueIndex(value); + } updateStats(value, time); } @@ -340,8 +350,10 @@ public abstract class Statistics<T> { if (time > this.endTime) { endTime = time; } - updateStepRegress(time); - updateValueIndex(value); + if (TSFileDescriptor.getInstance().getConfig().isWriteM4LSM()) { + updateStepRegress(time); + updateValueIndex(value); + } updateStats(value, time); } @@ -354,8 +366,10 @@ public abstract class Statistics<T> { if (time > this.endTime) { endTime = time; } - updateStepRegress(time); - updateValueIndex(value); + if (TSFileDescriptor.getInstance().getConfig().isWriteM4LSM()) { + updateStepRegress(time); + updateValueIndex(value); + } updateStats(value, time); } @@ -392,8 +406,10 @@ public abstract class Statistics<T> { if (time[batchSize - 1] > this.endTime) { endTime = time[batchSize - 1]; } - updateStepRegress(time, batchSize); - updateValueIndex(values, batchSize); + if (TSFileDescriptor.getInstance().getConfig().isWriteM4LSM()) { + updateStepRegress(time, batchSize); + updateValueIndex(values, batchSize); + } updateStats(values, time, batchSize); } @@ -406,8 +422,10 @@ public abstract class Statistics<T> { if (time[batchSize - 1] > this.endTime) { endTime = time[batchSize - 1]; } - updateStepRegress(time, batchSize); - updateValueIndex(values, batchSize); + if (TSFileDescriptor.getInstance().getConfig().isWriteM4LSM()) { + updateStepRegress(time, batchSize); + updateValueIndex(values, batchSize); + } updateStats(values, time, batchSize); } @@ -420,8 +438,10 @@ public abstract class Statistics<T> { if (time[batchSize - 1] > this.endTime) { endTime = time[batchSize - 1]; } - updateStepRegress(time, batchSize); - updateValueIndex(values, batchSize); + if (TSFileDescriptor.getInstance().getConfig().isWriteM4LSM()) { + updateStepRegress(time, batchSize); + updateValueIndex(values, batchSize); + } updateStats(values, time, batchSize); } @@ -434,8 +454,10 @@ public abstract class Statistics<T> { if (time[batchSize - 1] > this.endTime) { endTime = time[batchSize - 1]; } - updateStepRegress(time, batchSize); - updateValueIndex(values, batchSize); + if (TSFileDescriptor.getInstance().getConfig().isWriteM4LSM()) { + updateStepRegress(time, batchSize); + updateValueIndex(values, batchSize); + } updateStats(values, time, batchSize); } @@ -601,8 +623,10 @@ public abstract class Statistics<T> { statistics.setStartTime(ReadWriteIOUtils.readLong(buffer)); statistics.setEndTime(ReadWriteIOUtils.readLong(buffer)); statistics.deserialize(buffer); - statistics.deserializeStepRegress(buffer); - statistics.deserializeValueIndex(buffer); + if (TSFileDescriptor.getInstance().getConfig().isWriteM4LSM()) { + statistics.deserializeStepRegress(buffer); + statistics.deserializeValueIndex(buffer); + } statistics.isEmpty = false; return statistics; }
