This is an automated email from the ASF dual-hosted git repository. leirui pushed a commit to branch research/M4-visualization in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit a5487008125987d7004b056d5365f54bbf69b11c Author: Lei Rui <[email protected]> AuthorDate: Thu Jul 13 20:45:21 2023 +0800 add --- .../resources/conf/iotdb-engine.properties | 2 + .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 37 +++-- .../apache/iotdb/db/integration/m4/MyTest1.java | 8 +- .../apache/iotdb/db/integration/m4/MyTest2.java | 2 + .../apache/iotdb/db/integration/m4/MyTest3.java | 2 + .../apache/iotdb/db/integration/m4/MyTest4.java | 2 + .../apache/iotdb/db/integration/m4/MyTest5.java | 2 + .../apache/iotdb/db/integration/m4/MyTmpTest.java | 2 + .../iotdb/tsfile/common/conf/TSFileConfig.java | 158 ++++++++++++++++----- .../iotdb/tsfile/common/conf/TSFileDescriptor.java | 21 +-- .../file/metadata/statistics/ValueIndex.java | 10 +- 11 files changed, 177 insertions(+), 69 deletions(-) diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties index 48e720a2995..8b202feffaf 100644 --- a/server/src/assembly/resources/conf/iotdb-engine.properties +++ b/server/src/assembly/resources/conf/iotdb-engine.properties @@ -614,6 +614,8 @@ use_Mad=false use_ValueIndex=true +error_Param=1 + # group_size_in_byte=134217728 # The memory size for each series writer to pack page, default value is 64KB diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index f40296e82b1..2bc2f557910 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -18,6 +18,16 @@ */ package org.apache.iotdb.db.conf; +import com.google.common.net.InetAddresses; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.net.InetAddress; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.UnknownHostException; +import java.util.Properties; import org.apache.iotdb.db.conf.directories.DirectoryManager; import org.apache.iotdb.db.engine.StorageEngine; import org.apache.iotdb.db.engine.compaction.CompactionStrategy; @@ -27,21 +37,9 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.fileSystem.FSType; import org.apache.iotdb.tsfile.utils.FilePathUtils; - -import com.google.common.net.InetAddresses; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.net.InetAddress; -import java.net.MalformedURLException; -import java.net.URL; -import java.net.UnknownHostException; -import java.util.Properties; - public class IoTDBDescriptor { private static final Logger logger = LoggerFactory.getLogger(IoTDBDescriptor.class); @@ -107,7 +105,9 @@ public class IoTDBDescriptor { } } - /** load an property file and set TsfileDBConfig variables. */ + /** + * load an property file and set TsfileDBConfig variables. + */ @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning private void loadProps() { URL url = getPropsUrl(); @@ -919,6 +919,13 @@ public class IoTDBDescriptor { properties.getProperty( "use_Mad", Boolean.toString(TSFileDescriptor.getInstance().getConfig().isUseMad())))); + TSFileDescriptor.getInstance() + .getConfig() + .setErrorParam( + Integer.parseInt( + properties.getProperty( + "error_Param", + Integer.toString(TSFileDescriptor.getInstance().getConfig().getErrorParam())))); TSFileDescriptor.getInstance() .getConfig() .setUseValueIndex( @@ -1315,7 +1322,9 @@ public class IoTDBDescriptor { } } - /** Get default encode algorithm by data type */ + /** + * Get default encode algorithm by data type + */ public TSEncoding getDefaultEncodingByType(TSDataType dataType) { switch (dataType) { case BOOLEAN: diff --git a/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest1.java b/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest1.java index db948cd3da0..6953a73198c 100644 --- a/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest1.java +++ b/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest1.java @@ -72,12 +72,14 @@ public class MyTest1 { originalEnableCPV = config.isEnableCPV(); config.setEnableCPV(true); // CPV - originalUseChunkIndex = TSFileDescriptor.getInstance().getConfig().isUseTimeIndex(); - TSFileDescriptor.getInstance().getConfig().setUseTimeIndex(false); +// originalUseChunkIndex = TSFileDescriptor.getInstance().getConfig().isUseTimeIndex(); +// TSFileDescriptor.getInstance().getConfig().setUseTimeIndex(false); originalUseMad = TSFileDescriptor.getInstance().getConfig().isUseMad(); TSFileDescriptor.getInstance().getConfig().setUseMad(true); + TSFileDescriptor.getInstance().getConfig().setErrorParam(10); + EnvironmentUtils.envSetUp(); Class.forName(Config.JDBC_DRIVER_NAME); config.setTimestampPrecision("ms"); @@ -88,7 +90,7 @@ public class MyTest1 { EnvironmentUtils.cleanEnv(); config.setCompactionStrategy(originalCompactionStrategy); config.setEnableCPV(originalEnableCPV); - TSFileDescriptor.getInstance().getConfig().setUseTimeIndex(originalUseChunkIndex); +// TSFileDescriptor.getInstance().getConfig().setUseTimeIndex(originalUseChunkIndex); TSFileDescriptor.getInstance().getConfig().setUseMad(originalUseMad); } diff --git a/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest2.java b/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest2.java index 134fddbb2f6..be03ab8fa65 100644 --- a/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest2.java +++ b/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest2.java @@ -86,6 +86,8 @@ public class MyTest2 { true); // this test cannot be false, as the expected answer for bottomTime and topTime can // be different config.setTimestampPrecision("ms"); + + TSFileDescriptor.getInstance().getConfig().setErrorParam(10); } @After diff --git a/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest3.java b/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest3.java index fb5c514094e..1c9715b4e81 100644 --- a/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest3.java +++ b/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest3.java @@ -75,6 +75,8 @@ public class MyTest3 { EnvironmentUtils.envSetUp(); Class.forName(Config.JDBC_DRIVER_NAME); config.setTimestampPrecision("ms"); + + TSFileDescriptor.getInstance().getConfig().setErrorParam(10); } @After diff --git a/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest4.java b/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest4.java index db38c7e4f35..745c400e488 100644 --- a/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest4.java +++ b/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest4.java @@ -75,6 +75,8 @@ public class MyTest4 { EnvironmentUtils.envSetUp(); Class.forName(Config.JDBC_DRIVER_NAME); config.setTimestampPrecision("ms"); + + TSFileDescriptor.getInstance().getConfig().setErrorParam(10); } @After diff --git a/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest5.java b/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest5.java index 2f406f1153c..aa589c094ce 100644 --- a/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest5.java +++ b/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTest5.java @@ -81,6 +81,8 @@ public class MyTest5 { EnvironmentUtils.envSetUp(); Class.forName(Config.JDBC_DRIVER_NAME); config.setTimestampPrecision("ms"); + + TSFileDescriptor.getInstance().getConfig().setErrorParam(10); } @After diff --git a/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTmpTest.java b/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTmpTest.java index 98419762bdd..3dc01291b96 100644 --- a/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTmpTest.java +++ b/server/src/test/java/org/apache/iotdb/db/integration/m4/MyTmpTest.java @@ -80,6 +80,8 @@ public class MyTmpTest { EnvironmentUtils.envSetUp(); Class.forName(Config.JDBC_DRIVER_NAME); config.setTimestampPrecision("ms"); + + TSFileDescriptor.getInstance().getConfig().setErrorParam(10); } @After diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java index 26e518b865d..4c212eb4bc4 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java @@ -18,13 +18,14 @@ */ package org.apache.iotdb.tsfile.common.conf; -import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; -import org.apache.iotdb.tsfile.fileSystem.FSType; - import java.io.Serializable; import java.nio.charset.Charset; +import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; +import org.apache.iotdb.tsfile.fileSystem.FSType; -/** TSFileConfig is a configure class. Every variables is public and has default value. */ +/** + * TSFileConfig is a configure class. Every variables is public and has default value. + */ public class TSFileConfig implements Serializable { private boolean useStatistics = true; @@ -35,7 +36,11 @@ public class TSFileConfig implements Serializable { private boolean useMad = false; - /** encoding configuration */ + private int errorParam = 1; + + /** + * encoding configuration + */ public static final int RLE_MIN_REPEATED_NUM = 8; public static final int RLE_MAX_REPEATED_NUM = 0x7FFF; @@ -57,7 +62,9 @@ public class TSFileConfig implements Serializable { public static final float GORILLA_ENCODING_ENDING_FLOAT = Float.NaN; public static final double GORILLA_ENCODING_ENDING_DOUBLE = Double.NaN; - /** String encoder with UTF-8 encodes a character to at most 4 bytes. */ + /** + * String encoder with UTF-8 encodes a character to at most 4 bytes. + */ public static final int BYTE_SIZE_PER_CHAR = 4; public static final String STRING_ENCODING = "UTF-8"; @@ -66,29 +73,49 @@ public class TSFileConfig implements Serializable { public static final String MAGIC_STRING = "TsFile"; public static final String VERSION_NUMBER_V2 = "000002"; public static final String VERSION_NUMBER_V1 = "000001"; - /** version number is changed to use 1 byte to represent since version 3 */ + /** + * version number is changed to use 1 byte to represent since version 3 + */ public static final byte VERSION_NUMBER = 0x03; - /** Bloom filter constrain */ + /** + * Bloom filter constrain + */ public static final double MIN_BLOOM_FILTER_ERROR_RATE = 0.01; public static final double MAX_BLOOM_FILTER_ERROR_RATE = 0.1; - /** The primitive array capacity threshold. */ + /** + * The primitive array capacity threshold. + */ public static final int ARRAY_CAPACITY_THRESHOLD = 1000; - /** Memory size threshold for flushing to disk, default value is 128MB. */ + /** + * Memory size threshold for flushing to disk, default value is 128MB. + */ private int groupSizeInByte = 128 * 1024 * 1024; - /** The memory size for each series writer to pack page, default value is 64KB. */ + /** + * The memory size for each series writer to pack page, default value is 64KB. + */ private int pageSizeInByte = 64 * 1024; - /** The maximum number of data points in a page, default value is 1024 * 1024. */ + /** + * The maximum number of data points in a page, default value is 1024 * 1024. + */ private int maxNumberOfPointsInPage = 1024 * 1024; - /** The maximum degree of a metadataIndex node, default value is 256 */ + /** + * The maximum degree of a metadataIndex node, default value is 256 + */ private int maxDegreeOfIndexNode = 256; - /** Data type for input timestamp, TsFile supports INT32 or INT64. */ + /** + * Data type for input timestamp, TsFile supports INT32 or INT64. + */ private String timeSeriesDataType = "INT64"; - /** Max length limitation of input string. */ + /** + * Max length limitation of input string. + */ private int maxStringLength = 128; - /** Floating-point precision. */ + /** + * Floating-point precision. + */ private int floatPrecision = 2; /** * Encoder of time column, TsFile supports TS_2DIFF, PLAIN and RLE(run-length encoding) Default @@ -102,39 +129,73 @@ public class TSFileConfig implements Serializable { * supports PLAIN. */ private String valueEncoder = "PLAIN"; - /** Default bit width of RLE encoding is 8. */ + /** + * Default bit width of RLE encoding is 8. + */ private int rleBitWidth = 8; - /** Default block size of two-diff. delta encoding is 128 */ + /** + * Default block size of two-diff. delta encoding is 128 + */ private int deltaBlockSize = 128; - /** Default frequency type is SINGLE_FREQ. */ + /** + * Default frequency type is SINGLE_FREQ. + */ private String freqType = "SINGLE_FREQ"; - /** Default PLA max error is 100. */ + /** + * Default PLA max error is 100. + */ private double plaMaxError = 100; - /** Default SDT max error is 100. */ + /** + * Default SDT max error is 100. + */ private double sdtMaxError = 100; - /** Default DFT satisfy rate is 0.1 */ + /** + * Default DFT satisfy rate is 0.1 + */ private double dftSatisfyRate = 0.1; - /** Data compression method, TsFile supports UNCOMPRESSED, SNAPPY or LZ4. */ + /** + * Data compression method, TsFile supports UNCOMPRESSED, SNAPPY or LZ4. + */ private CompressionType compressor = CompressionType.UNCOMPRESSED; - /** Line count threshold for checking page memory occupied size. */ + /** + * Line count threshold for checking page memory occupied size. + */ private int pageCheckSizeThreshold = 100; - /** Default endian value is BIG_ENDIAN. */ + /** + * Default endian value is BIG_ENDIAN. + */ private String endian = "BIG_ENDIAN"; - /** Default storage is in local file system */ + /** + * Default storage is in local file system + */ private FSType TSFileStorageFs = FSType.LOCAL; - /** Default core-site.xml file path is /etc/hadoop/conf/core-site.xml */ + /** + * Default core-site.xml file path is /etc/hadoop/conf/core-site.xml + */ private String coreSitePath = "/etc/hadoop/conf/core-site.xml"; - /** Default hdfs-site.xml file path is /etc/hadoop/conf/hdfs-site.xml */ + /** + * Default hdfs-site.xml file path is /etc/hadoop/conf/hdfs-site.xml + */ private String hdfsSitePath = "/etc/hadoop/conf/hdfs-site.xml"; - /** Default hdfs ip is localhost */ + /** + * Default hdfs ip is localhost + */ private String hdfsIp = "localhost"; - /** Default hdfs port is 9000 */ + /** + * Default hdfs port is 9000 + */ private String hdfsPort = "9000"; - /** Default DFS NameServices is hdfsnamespace */ + /** + * Default DFS NameServices is hdfsnamespace + */ private String dfsNameServices = "hdfsnamespace"; - /** Default DFS HA name nodes are nn1 and nn2 */ + /** + * Default DFS HA name nodes are nn1 and nn2 + */ private String dfsHaNamenodes = "nn1,nn2"; - /** Default DFS HA automatic failover is enabled */ + /** + * Default DFS HA automatic failover is enabled + */ private boolean dfsHaAutomaticFailoverEnabled = true; /** * Default DFS client failover proxy provider is @@ -142,18 +203,29 @@ public class TSFileConfig implements Serializable { */ private String dfsClientFailoverProxyProvider = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"; - /** whether use kerberos to authenticate hdfs */ + /** + * whether use kerberos to authenticate hdfs + */ private boolean useKerberos = false; - /** full path of kerberos keytab file */ + /** + * full path of kerberos keytab file + */ private String kerberosKeytabFilePath = "/path"; - /** kerberos pricipal */ + /** + * kerberos pricipal + */ private String kerberosPrincipal = "principal"; - /** The acceptable error rate of bloom filter */ + /** + * The acceptable error rate of bloom filter + */ private double bloomFilterErrorRate = 0.05; - /** The amount of data iterate each time */ + /** + * The amount of data iterate each time + */ private int batchSize = 1000; - public TSFileConfig() {} + public TSFileConfig() { + } public boolean isUseStatistics() { return useStatistics; @@ -179,6 +251,14 @@ public class TSFileConfig implements Serializable { this.useMad = useMad; } + public int getErrorParam() { + return errorParam; + } + + public void setErrorParam(int errorParam) { + this.errorParam = errorParam; + } + public boolean isUseValueIndex() { return useValueIndex; } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java index 4394fd4ce67..0e0f6222a50 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java @@ -19,12 +19,6 @@ package org.apache.iotdb.tsfile.common.conf; -import org.apache.iotdb.tsfile.common.constant.TsFileConstant; -import org.apache.iotdb.tsfile.utils.Loader; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; @@ -33,8 +27,14 @@ import java.io.InputStream; import java.net.URL; import java.util.Properties; import java.util.Set; +import org.apache.iotdb.tsfile.common.constant.TsFileConstant; +import org.apache.iotdb.tsfile.utils.Loader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -/** TSFileDescriptor is used to load TSFileConfig and provide configure information. */ +/** + * TSFileDescriptor is used to load TSFileConfig and provide configure information. + */ public class TSFileDescriptor { private static final Logger logger = LoggerFactory.getLogger(TSFileDescriptor.class); @@ -70,7 +70,9 @@ public class TSFileDescriptor { return Loader.getResource(filename, classLoader); } - /** load an .properties file and set TSFileConfig variables */ + /** + * load an .properties file and set TSFileConfig variables + */ private void loadProps() { InputStream inputStream; String url = System.getProperty(TsFileConstant.TSFILE_CONF, null); @@ -114,6 +116,9 @@ public class TSFileDescriptor { conf.setUseMad( Boolean.parseBoolean( properties.getProperty("use_Mad", Boolean.toString(conf.isUseMad())))); + conf.setErrorParam( + Integer.parseInt( + properties.getProperty("error_Param", Integer.toString(conf.getErrorParam())))); conf.setUseValueIndex( Boolean.parseBoolean( properties.getProperty("use_ValueIndex", Boolean.toString(conf.isUseValueIndex())))); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/ValueIndex.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/ValueIndex.java index 8b2063ce365..8a0dd9066a1 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/ValueIndex.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/ValueIndex.java @@ -1,19 +1,19 @@ package org.apache.iotdb.tsfile.file.metadata.statistics; +import java.util.ArrayList; +import java.util.List; +import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; import org.apache.iotdb.tsfile.encoding.encoder.PlainEncoder; import org.apache.iotdb.tsfile.encoding.encoder.SDTEncoder; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.read.common.ValuePoint; import org.apache.iotdb.tsfile.utils.PublicBAOS; - import org.eclipse.collections.impl.list.mutable.primitive.DoubleArrayList; import org.eclipse.collections.impl.list.mutable.primitive.IntArrayList; -import java.util.ArrayList; -import java.util.List; - public class ValueIndex { + int errorParam = TSFileDescriptor.getInstance().getConfig().getErrorParam(); private DoubleArrayList values = new DoubleArrayList(); public SDTEncoder sdtEncoder = new SDTEncoder(); public double errorBound = 0; // =std*2 =2*std.compDeviation @@ -76,7 +76,7 @@ public class ValueIndex { private void initForLearn() { this.stdDev = getStdDev(); this.errorBound = 2 * stdDev; - this.sdtEncoder.setCompDeviation(errorBound / 2); // equals stdDev is best + this.sdtEncoder.setCompDeviation(errorBound / 2 * errorParam); // equals stdDev is best } public void learn() {
