This is an automated email from the ASF dual-hosted git repository. leirui pushed a commit to branch research/LTS-visualization in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 7a8a1bdd0197731b4d68f8b4d82841ec2b48f002 Author: Lei Rui <[email protected]> AuthorDate: Sat Feb 3 22:01:39 2024 +0800 add write_convex_hull --- .../resources/conf/iotdb-engine.properties | 3 ++ .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 2 +- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 8 ++++ .../groupby/GroupByWithoutValueFilterDataSet.java | 5 +++ .../iotdb/db/integration/tri/MyTest_ILTS.java | 3 +- .../iotdb/tsfile/common/conf/TSFileConfig.java | 10 +++++ .../iotdb/tsfile/common/conf/TSFileDescriptor.java | 4 ++ .../file/metadata/statistics/Statistics.java | 45 ++++++++++++++++------ 8 files changed, 67 insertions(+), 13 deletions(-) diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties index 5ab21cc46b5..1e935c8bdba 100644 --- a/server/src/assembly/resources/conf/iotdb-engine.properties +++ b/server/src/assembly/resources/conf/iotdb-engine.properties @@ -627,6 +627,9 @@ enable_unseq_compaction=false #################### ### Configurations for tsfile-format #################### + +write_convex_hull=true + enableMinMaxLSM=false use_Statistics=true diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 4d3cc624750..765f9e2ed05 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -450,7 +450,7 @@ public class IoTDBConfig { this.numIterations = numIterations; } - private int numIterations = 2; + private int numIterations = 4; public boolean isAcc_avg() { return acc_avg; diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index afe5e484119..8f8fd7b31df 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -920,6 +920,14 @@ public class IoTDBDescriptor { } private void loadTsFileProps(Properties properties) { + TSFileDescriptor.getInstance() + .getConfig() + .setWriteConvexHull( + Boolean.parseBoolean( + properties.getProperty( + "write_convex_hull", + Boolean.toString( + TSFileDescriptor.getInstance().getConfig().isWriteConvexHull())))); TSFileDescriptor.getInstance() .getConfig() .setEnableMinMaxLSM( diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java index e71bc91f6c6..e674e4bf7bc 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java +++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java @@ -31,6 +31,7 @@ import org.apache.iotdb.db.query.aggregation.AggregateResult; import org.apache.iotdb.db.query.context.QueryContext; import org.apache.iotdb.db.query.factory.AggregateResultFactory; import org.apache.iotdb.db.query.filter.TsFileFilter; +import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.statistics.MinMaxInfo; import org.apache.iotdb.tsfile.read.common.IOMonitor2; @@ -663,6 +664,10 @@ public class GroupByWithoutValueFilterDataSet extends GroupByEngineDataSet { return new LocalGroupByExecutorTri_LTTB( path, allSensors, dataType, context, timeFilter, fileFilter, ascending); } else if (CONFIG.getEnableTri().equals("ILTS")) { + if (!TSFileDescriptor.getInstance().getConfig().isWriteConvexHull() + && CONFIG.isAcc_convex()) { + throw new QueryProcessException("ILTS use convex hull acceleration, which is not written!"); + } return new LocalGroupByExecutorTri_ILTS( path, allSensors, dataType, context, timeFilter, fileFilter, ascending); } else { diff --git a/server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_ILTS.java b/server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_ILTS.java index e7695b90e10..5f8dfde95ba 100644 --- a/server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_ILTS.java +++ b/server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_ILTS.java @@ -81,8 +81,9 @@ public class MyTest_ILTS { // config.setNumIterations(4); config.setAcc_avg(true); config.setAcc_rectangle(true); - config.setAcc_convex(true); + config.setAcc_convex(false); config.setAcc_iterRepeat(true); + TSFileDescriptor.getInstance().getConfig().setWriteConvexHull(false); config.setEnableCPV(false); TSFileDescriptor.getInstance().getConfig().setEnableMinMaxLSM(false); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java index c745982c5a4..3e90214d277 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java @@ -27,6 +27,8 @@ import java.nio.charset.Charset; /** TSFileConfig is a configure class. Every variables is public and has default value. */ public class TSFileConfig implements Serializable { + private boolean writeConvexHull = true; + private boolean enableMinMaxLSM = false; private boolean useStatistics = true; @@ -159,6 +161,14 @@ public class TSFileConfig implements Serializable { public TSFileConfig() {} + public boolean isWriteConvexHull() { + return writeConvexHull; + } + + public void setWriteConvexHull(boolean writeConvexHull) { + this.writeConvexHull = writeConvexHull; + } + public boolean isEnableMinMaxLSM() { return enableMinMaxLSM; } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java index 3aa322b4e89..e0d066a24fb 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java @@ -105,6 +105,10 @@ public class TSFileDescriptor { Properties properties = new Properties(); try { properties.load(inputStream); + conf.setWriteConvexHull( + Boolean.parseBoolean( + properties.getProperty( + "write_convex_hull", Boolean.toString(conf.isWriteConvexHull())))); conf.setEnableMinMaxLSM( Boolean.parseBoolean( properties.getProperty( diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java index 89a9ffe8c41..2b9fe6e2399 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java @@ -18,6 +18,7 @@ */ package org.apache.iotdb.tsfile.file.metadata.statistics; +import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; import org.apache.iotdb.tsfile.encoding.decoder.Decoder; import org.apache.iotdb.tsfile.encoding.decoder.PlainDecoder; import org.apache.iotdb.tsfile.exception.filter.StatisticsClassException; @@ -151,7 +152,9 @@ public abstract class Statistics<T> { // // serialize value index // byteLen += serializeValueIndex(outputStream, true); // serialize convex hull - byteLen += serializeConvexHull(outputStream, true); + if (TSFileDescriptor.getInstance().getConfig().isWriteConvexHull()) { + byteLen += serializeConvexHull(outputStream, true); + } return byteLen; } @@ -167,7 +170,9 @@ public abstract class Statistics<T> { // // serialize value index // byteLen += serializeValueIndex(outputStream, log); // serialize convex hull - byteLen += serializeConvexHull(outputStream, log); + if (TSFileDescriptor.getInstance().getConfig().isWriteConvexHull()) { + byteLen += serializeConvexHull(outputStream, log); + } return byteLen; } @@ -365,7 +370,9 @@ public abstract class Statistics<T> { // update time index // updateStepRegress(time); // updateValueIndex(value); - updateConvexHull(time, value); + if (TSFileDescriptor.getInstance().getConfig().isWriteConvexHull()) { + updateConvexHull(time, value); + } } /** @author Yuyuan Kang */ @@ -380,7 +387,9 @@ public abstract class Statistics<T> { updateStats(value, time); // updateStepRegress(time); // updateValueIndex(value); - updateConvexHull(time, value); + if (TSFileDescriptor.getInstance().getConfig().isWriteConvexHull()) { + updateConvexHull(time, value); + } } /** @author Yuyuan Kang */ @@ -395,7 +404,9 @@ public abstract class Statistics<T> { updateStats(value, time); // updateStepRegress(time); // updateValueIndex(value); - updateConvexHull(time, value); + if (TSFileDescriptor.getInstance().getConfig().isWriteConvexHull()) { + updateConvexHull(time, value); + } } /** @author Yuyuan Kang */ @@ -410,7 +421,9 @@ public abstract class Statistics<T> { updateStats(value, time); // updateStepRegress(time); // updateValueIndex(value); - updateConvexHull(time, value); + if (TSFileDescriptor.getInstance().getConfig().isWriteConvexHull()) { + updateConvexHull(time, value); + } } @Deprecated @@ -449,7 +462,9 @@ public abstract class Statistics<T> { updateStats(values, time, batchSize); // updateStepRegress(time, batchSize); // updateValueIndex(values, batchSize); - updateConvexHull(time, values, batchSize); + if (TSFileDescriptor.getInstance().getConfig().isWriteConvexHull()) { + updateConvexHull(time, values, batchSize); + } } /** @author Yuyuan Kang */ @@ -464,7 +479,9 @@ public abstract class Statistics<T> { updateStats(values, time, batchSize); // updateStepRegress(time, batchSize); // updateValueIndex(values, batchSize); - updateConvexHull(time, values, batchSize); + if (TSFileDescriptor.getInstance().getConfig().isWriteConvexHull()) { + updateConvexHull(time, values, batchSize); + } } /** @author Yuyuan Kang */ @@ -479,7 +496,9 @@ public abstract class Statistics<T> { updateStats(values, time, batchSize); // updateStepRegress(time, batchSize); // updateValueIndex(values, batchSize); - updateConvexHull(time, values, batchSize); + if (TSFileDescriptor.getInstance().getConfig().isWriteConvexHull()) { + updateConvexHull(time, values, batchSize); + } } /** @author Yuyuan Kang */ @@ -494,7 +513,9 @@ public abstract class Statistics<T> { updateStats(values, time, batchSize); // updateStepRegress(time, batchSize); // updateValueIndex(values, batchSize); - updateConvexHull(time, values, batchSize); + if (TSFileDescriptor.getInstance().getConfig().isWriteConvexHull()) { + updateConvexHull(time, values, batchSize); + } } @Deprecated @@ -689,7 +710,9 @@ public abstract class Statistics<T> { statistics.deserialize(buffer); // statistics.deserializeStepRegress(buffer); // statistics.deserializeValueIndex(buffer); - statistics.deserializeConvexHull(buffer); + if (TSFileDescriptor.getInstance().getConfig().isWriteConvexHull()) { + statistics.deserializeConvexHull(buffer); + } statistics.isEmpty = false; return statistics; }
