This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch update_config in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 59aa93f27446eb55039d901b774e6f6f508319d9 Author: HTHou <[email protected]> AuthorDate: Thu Jun 15 19:14:30 2023 +0800 remove freq --- docs/UserGuide/Data-Concept/Encoding.md | 22 +- docs/UserGuide/Reference/Common-Config-Manual.md | 17 -- docs/zh/UserGuide/Data-Concept/Encoding.md | 22 +- .../zh/UserGuide/Reference/Common-Config-Manual.md | 17 -- .../org/apache/iotdb/db/it/IoTDBEncodingIT.java | 59 ---- iotdb-client/client-cpp/src/main/Session.h | 1 - .../client-py/iotdb/utils/IoTDBConstants.py | 1 - .../util/TSFileConfigUtilCompletenessTest.java | 2 - .../org/apache/iotdb/db/utils/SchemaUtils.java | 2 - tsfile/pom.xml | 4 - .../iotdb/tsfile/common/conf/TSFileConfig.java | 20 -- .../iotdb/tsfile/encoding/decoder/Decoder.java | 2 - .../iotdb/tsfile/encoding/decoder/FreqDecoder.java | 144 ---------- .../iotdb/tsfile/encoding/encoder/FreqEncoder.java | 317 --------------------- .../tsfile/encoding/encoder/TSEncodingBuilder.java | 64 ----- .../tsfile/file/metadata/enums/TSEncoding.java | 3 +- .../apache/iotdb/tsfile/utils/BitConstructor.java | 94 ------ .../org/apache/iotdb/tsfile/utils/BitReader.java | 70 ----- .../tsfile/encoding/decoder/FreqDecoderTest.java | 161 ----------- 19 files changed, 17 insertions(+), 1005 deletions(-) diff --git a/docs/UserGuide/Data-Concept/Encoding.md b/docs/UserGuide/Data-Concept/Encoding.md index 6f479df368c..b82aae3edb5 100644 --- a/docs/UserGuide/Data-Concept/Encoding.md +++ b/docs/UserGuide/Data-Concept/Encoding.md @@ -54,12 +54,6 @@ Usage restrictions: When using GORILLA to encode INT32 data, you need to ensure DICTIONARY encoding is lossless. It is suitable for TEXT data with low cardinality (i.e. low number of distinct values). It is not recommended to use it for high-cardinality data. -* FREQ - -FREQ encoding is lossy. Based on the idea of transform coding, it transforms the time sequence to the frequency domain and only reserve part of the frequency components with high energy. Thus, it greatly improves the space efficiency with little accuracy loss. It is suitable for data with high energy concentration (especially those with obvious periodicity), not suitable for data with uniformly distributed energy (such as white noise). - -> There are two parameters of FREQ encoding in the configuration file: `freq_snr` defines the signal-noise-ratio (SNR). There is a mathematical relationship between SNR and NRMSE as $NRMSE = 10^{-SNR/20}$. Both the compression ratio and accuracy loss decrease when it increases. `freq_block_size` defines the data size in a time-frequency transformation. It is not recommended to modify the default value. The detailed experimental results and analysis of the influences of parameters are in [...] - * ZIGZAG ZIGZAG encoding maps signed integers to unsigned integers so that numbers with a small absolute value (for instance, -1) have a small variant encoded value too. It does this in a way that "zig-zags" back and forth through the positive and negative integers. @@ -84,14 +78,14 @@ The five encodings described in the previous sections are applicable to differen The correspondence between the data type and its supported encodings is summarized in the Table below. -| Data Type | Supported Encoding | -|:---------:|:-----------------------------------------------------------------:| -| BOOLEAN | PLAIN, RLE | -| INT32 | PLAIN, RLE, TS_2DIFF, GORILLA, FREQ, ZIGZAG, CHIMP, SPRINTZ, RLBE | -| INT64 | PLAIN, RLE, TS_2DIFF, GORILLA, FREQ, ZIGZAG, CHIMP, SPRINTZ, RLBE | -| FLOAT | PLAIN, RLE, TS_2DIFF, GORILLA, FREQ, CHIMP, SPRINTZ, RLBE | -| DOUBLE | PLAIN, RLE, TS_2DIFF, GORILLA, FREQ, CHIMP, SPRINTZ, RLBE | -| TEXT | PLAIN, DICTIONARY | +| Data Type | Supported Encoding | +|:---------:|:-----------------------------------------------------------:| +| BOOLEAN | PLAIN, RLE | +| INT32 | PLAIN, RLE, TS_2DIFF, GORILLA, ZIGZAG, CHIMP, SPRINTZ, RLBE | +| INT64 | PLAIN, RLE, TS_2DIFF, GORILLA, ZIGZAG, CHIMP, SPRINTZ, RLBE | +| FLOAT | PLAIN, RLE, TS_2DIFF, GORILLA, CHIMP, SPRINTZ, RLBE | +| DOUBLE | PLAIN, RLE, TS_2DIFF, GORILLA, CHIMP, SPRINTZ, RLBE | +| TEXT | PLAIN, DICTIONARY | When the data type specified by the user does not correspond to the encoding method, the system will prompt an error. diff --git a/docs/UserGuide/Reference/Common-Config-Manual.md b/docs/UserGuide/Reference/Common-Config-Manual.md index 0564bc1f6da..0b16964cbee 100644 --- a/docs/UserGuide/Reference/Common-Config-Manual.md +++ b/docs/UserGuide/Reference/Common-Config-Manual.md @@ -1264,23 +1264,6 @@ Different configuration parameters take effect in the following three ways: | Default | 0.05 | | Effective | After restarting system | -* freq\_snr - -| Name | freq\_snr | -| :---------: | :---------------------------------------------- | -| Description | Signal-noise-ratio (SNR) of lossy FREQ encoding | -| Type | Double | -| Default | 40.0 | -| Effective | hot-load | - -* freq\_block\_size - -|Name| freq\_block\_size | -|:---:|:---| -|Description| Block size of FREQ encoding. In other words, the number of data points in a time-frequency transformation. To speed up the encoding, it is recommended to be the power of 2. | -|Type|int32| -|Default| 1024 | -|Effective|hot-load| ### Authorization Configuration diff --git a/docs/zh/UserGuide/Data-Concept/Encoding.md b/docs/zh/UserGuide/Data-Concept/Encoding.md index e3515cfd291..5da4a73fe92 100644 --- a/docs/zh/UserGuide/Data-Concept/Encoding.md +++ b/docs/zh/UserGuide/Data-Concept/Encoding.md @@ -53,12 +53,6 @@ GORILLA 编码是一种无损编码，它比较适合编码前后值比较接近 字典编码是一种无损编码。它适合编码基数小的数据（即数据去重后唯一值数量小）。不推荐用于基数大的数据。 -* 频域编码 （FREQ） - -频域编码是一种有损编码，它基于变换编码的思想，将时序数据变换为频域，仅保留部分高能量的频域分量，以少许的精度损失为代价大幅提高空间效率。该编码适合于频域能量分布较为集中的数据（特别是具有明显周期性的数据），不适合能量分布均匀的数据（如白噪声）。 - -> 频域编码在配置文件中包括两个参数：`freq_snr`指定了编码的信噪比（与标准均方根误差的关系为$NRMSE=10^{-SNR/20}$），该参数增大会同时降低压缩比和精度损失，请根据实际应用的需要进行设置；`freq_block_size`指定了编码进行时频域变换的分组大小，推荐不对默认值进行修改。参数影响的实验结果和分析详见设计文档。 - * ZIGZAG 编码 ZigZag编码将有符号整型映射到无符号整型，适合比较小的整数。 @@ -84,14 +78,14 @@ RLBE编码是一种无损编码，将差分编码，位填充编码，游程长 前文介绍的五种编码适用于不同的数据类型，若对应关系错误，则无法正确创建时间序列。数据类型与支持其编码的编码方式对应关系总结如下表所示。 -| 数据类型 | 支持的编码 | -|:---------:|:-----------------------------------------------------------------:| -| BOOLEAN | PLAIN, RLE | -| INT32 | PLAIN, RLE, TS_2DIFF, GORILLA, FREQ, ZIGZAG, CHIMP, SPRINTZ, RLBE | -| INT64 | PLAIN, RLE, TS_2DIFF, GORILLA, FREQ, ZIGZAG, CHIMP, SPRINTZ, RLBE | -| FLOAT | PLAIN, RLE, TS_2DIFF, GORILLA, FREQ, CHIMP, SPRINTZ, RLBE | -| DOUBLE | PLAIN, RLE, TS_2DIFF, GORILLA, FREQ, CHIMP, SPRINTZ, RLBE | -| TEXT | PLAIN, DICTIONARY | +| 数据类型 | 支持的编码 | +|:---------:|:-----------------------------------------------------------:| +| BOOLEAN | PLAIN, RLE | +| INT32 | PLAIN, RLE, TS_2DIFF, GORILLA, ZIGZAG, CHIMP, SPRINTZ, RLBE | +| INT64 | PLAIN, RLE, TS_2DIFF, GORILLA, ZIGZAG, CHIMP, SPRINTZ, RLBE | +| FLOAT | PLAIN, RLE, TS_2DIFF, GORILLA, CHIMP, SPRINTZ, RLBE | +| DOUBLE | PLAIN, RLE, TS_2DIFF, GORILLA, CHIMP, SPRINTZ, RLBE | +| TEXT | PLAIN, DICTIONARY | 当用户输入的数据类型与编码方式不对应时，系统会提示错误。如下所示，二阶差分编码不支持布尔类型： diff --git a/docs/zh/UserGuide/Reference/Common-Config-Manual.md b/docs/zh/UserGuide/Reference/Common-Config-Manual.md index a8c1a716fbf..d6748669134 100644 --- a/docs/zh/UserGuide/Reference/Common-Config-Manual.md +++ b/docs/zh/UserGuide/Reference/Common-Config-Manual.md @@ -1312,23 +1312,6 @@ IoTDB ConfigNode 和 DataNode 的公共配置参数位于 `conf` 目录下。 |默认值| 1 | |改后生效方式|热加载| -* freq\_snr - -| 名字 | freq\_snr | -| :----------: | :--------------------- | -| 描述 | 有损的FREQ编码的信噪比 | -| 类型 | Double | -| 默认值 | 40.0 | -| 改后生效方式 | 热加载 | - -* freq\_block\_size - -|名字| freq\_block\_size | -|:---:|:---| -|描述| FREQ编码的块大小，即一次时频域变换的数据点个数。为了加快编码速度，建议将其设置为2的幂次。 | -|类型|int32| -|默认值| 1024 | -|改后生效方式|热加载| #### 授权配置 diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBEncodingIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBEncodingIT.java index 71e408c5312..aa7b7024677 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBEncodingIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBEncodingIT.java @@ -36,7 +36,6 @@ import java.sql.SQLException; import java.sql.Statement; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @@ -279,64 +278,6 @@ public class IoTDBEncodingIT { } } - @Test - public void testSetTimeEncoderRegularAndValueEncoderFREQ() { - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { - statement.execute( - "CREATE TIMESERIES root.db_0.tab0.salary WITH DATATYPE=INT64,ENCODING=FREQ"); - statement.execute("insert into root.db_0.tab0(time,salary) values(1,1100)"); - statement.execute("insert into root.db_0.tab0(time,salary) values(2,1200)"); - statement.execute("insert into root.db_0.tab0(time,salary) values(3,1300)"); - statement.execute("insert into root.db_0.tab0(time,salary) values(4,1400)"); - statement.execute("flush"); - - int[] groundtruth = new int[] {1100, 1200, 1300, 1400}; - int[] result = new int[4]; - try (ResultSet resultSet = statement.executeQuery("select * from root.db_0.tab0")) { - int index = 0; - while (resultSet.next()) { - int salary = resultSet.getInt("root.db_0.tab0.salary"); - result[index] = salary; - index++; - } - assertTrue(SNR(groundtruth, result, groundtruth.length) > 40); - } - } catch (Exception e) { - e.printStackTrace(); - fail(); - } - } - - @Test - public void testSetTimeEncoderRegularAndValueEncoderFREQOutofOrder() { - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { - statement.execute( - "CREATE TIMESERIES root.db_0.tab0.salary WITH DATATYPE=INT64,ENCODING=FREQ"); - statement.execute("insert into root.db_0.tab0(time,salary) values(1,1200)"); - statement.execute("insert into root.db_0.tab0(time,salary) values(2,1100)"); - statement.execute("insert into root.db_0.tab0(time,salary) values(7,1000)"); - statement.execute("insert into root.db_0.tab0(time,salary) values(4,2200)"); - statement.execute("flush"); - - int[] groundtruth = new int[] {1200, 1100, 2200, 1000}; - int[] result = new int[4]; - try (ResultSet resultSet = statement.executeQuery("select * from root.db_0.tab0")) { - int index = 0; - while (resultSet.next()) { - int salary = resultSet.getInt("root.db_0.tab0.salary"); - result[index] = salary; - index++; - } - assertTrue(SNR(groundtruth, result, groundtruth.length) > 40); - } - } catch (Exception e) { - e.printStackTrace(); - fail(); - } - } - @Test public void testSetTimeEncoderRegularAndValueEncoderDictionary() { try (Connection connection = EnvFactory.getEnv().getConnection(); diff --git a/iotdb-client/client-cpp/src/main/Session.h b/iotdb-client/client-cpp/src/main/Session.h index 11341b163be..d57aad88655 100644 --- a/iotdb-client/client-cpp/src/main/Session.h +++ b/iotdb-client/client-cpp/src/main/Session.h @@ -180,7 +180,6 @@ namespace TSEncoding { REGULAR = (char) 7, GORILLA = (char) 8, ZIGZAG = (char) 9, - FREQ = (char) 10, CHIMP = (char) 11, SPRINTZ = (char) 12, RLBE = (char) 13 diff --git a/iotdb-client/client-py/iotdb/utils/IoTDBConstants.py b/iotdb-client/client-py/iotdb/utils/IoTDBConstants.py index 36a8ceb66bc..fac308b7556 100644 --- a/iotdb-client/client-py/iotdb/utils/IoTDBConstants.py +++ b/iotdb-client/client-py/iotdb/utils/IoTDBConstants.py @@ -60,7 +60,6 @@ class TSEncoding(Enum): REGULAR = 7 GORILLA = 8 ZIGZAG = 9 - FREQ = 10 CHIMP = 11 SPRINTZ = 12 RLBE = 13 diff --git a/iotdb-connector/flink-tsfile-connector/src/test/java/org/apache/iotdb/flink/util/TSFileConfigUtilCompletenessTest.java b/iotdb-connector/flink-tsfile-connector/src/test/java/org/apache/iotdb/flink/util/TSFileConfigUtilCompletenessTest.java index c209271f82c..5189faf6506 100644 --- a/iotdb-connector/flink-tsfile-connector/src/test/java/org/apache/iotdb/flink/util/TSFileConfigUtilCompletenessTest.java +++ b/iotdb-connector/flink-tsfile-connector/src/test/java/org/apache/iotdb/flink/util/TSFileConfigUtilCompletenessTest.java @@ -73,8 +73,6 @@ public class TSFileConfigUtilCompletenessTest { "setTSFileStorageFs", "setUseKerberos", "setValueEncoder", - "setFreqEncodingSNR", - "setFreqEncodingBlockSize", "setMaxTsBlockLineNumber", "setMaxTsBlockSizeInBytes", "setPatternMatchingThreshold", diff --git a/server/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java index 267aa0e381e..5e576fe11bd 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java @@ -55,7 +55,6 @@ public class SchemaUtils { intSet.add(TSEncoding.TS_2DIFF); intSet.add(TSEncoding.GORILLA); intSet.add(TSEncoding.ZIGZAG); - intSet.add(TSEncoding.FREQ); intSet.add(TSEncoding.CHIMP); intSet.add(TSEncoding.SPRINTZ); intSet.add(TSEncoding.RLBE); @@ -69,7 +68,6 @@ public class SchemaUtils { floatSet.add(TSEncoding.TS_2DIFF); floatSet.add(TSEncoding.GORILLA_V1); floatSet.add(TSEncoding.GORILLA); - floatSet.add(TSEncoding.FREQ); floatSet.add(TSEncoding.CHIMP); floatSet.add(TSEncoding.SPRINTZ); floatSet.add(TSEncoding.RLBE); diff --git a/tsfile/pom.xml b/tsfile/pom.xml index e651a449237..0d3ca0ce719 100644 --- a/tsfile/pom.xml +++ b/tsfile/pom.xml @@ -61,10 +61,6 @@ <groupId>org.lz4</groupId> <artifactId>lz4-java</artifactId> </dependency> - <dependency> - <groupId>com.github.wendykierp</groupId> - <artifactId>JTransforms</artifactId> - </dependency> <dependency> <groupId>com.google.code.gson</groupId> <artifactId>gson</artifactId> diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java index 1442a1d418f..9c05a42b765 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java @@ -109,10 +109,6 @@ public class TSFileConfig implements Serializable { private double sdtMaxError = 100; /** Default DFT satisfy rate is 0.1 */ private double dftSatisfyRate = 0.1; - /** Default SNR for FREQ encoding is 40dB. */ - private double freqEncodingSNR = 40; - /** Default block size for FREQ encoding is 1024. */ - private int freqEncodingBlockSize = 1024; /** Data compression method, TsFile supports UNCOMPRESSED, SNAPPY, ZSTD or LZ4. */ private CompressionType compressor = CompressionType.SNAPPY; /** Line count threshold for checking page memory occupied size. */ @@ -442,22 +438,6 @@ public class TSFileConfig implements Serializable { this.batchSize = batchSize; } - public double getFreqEncodingSNR() { - return freqEncodingSNR; - } - - public void setFreqEncodingSNR(double freqEncodingSNR) { - this.freqEncodingSNR = freqEncodingSNR; - } - - public int getFreqEncodingBlockSize() { - return freqEncodingBlockSize; - } - - public void setFreqEncodingBlockSize(int freqEncodingBlockSize) { - this.freqEncodingBlockSize = freqEncodingBlockSize; - } - public int getMaxTsBlockSizeInBytes() { return maxTsBlockSizeInBytes; } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/Decoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/Decoder.java index 4c4eda9dfcc..7a01964a733 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/Decoder.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/Decoder.java @@ -121,8 +121,6 @@ public abstract class Decoder { default: throw new TsFileDecodingException(String.format(ERROR_MSG, encoding, dataType)); } - case FREQ: - return new FreqDecoder(); case CHIMP: switch (dataType) { case FLOAT: diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/FreqDecoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/FreqDecoder.java deleted file mode 100644 index e4df75cb9d4..00000000000 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/FreqDecoder.java +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.tsfile.encoding.decoder; - -import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; -import org.apache.iotdb.tsfile.utils.BitReader; - -import org.jtransforms.dct.DoubleDCT_1D; - -import java.io.IOException; -import java.nio.ByteBuffer; - -public class FreqDecoder extends Decoder { - - private double data[]; - - private int readTotalCount = 0; - - private int nextReadIndex = 0; - - public FreqDecoder() { - super(TSEncoding.FREQ); - } - - @Override - public double readDouble(ByteBuffer buffer) { - if (nextReadIndex == readTotalCount) { - loadBlock(buffer); - nextReadIndex = 0; - } - return data[nextReadIndex++]; - } - - @Override - public float readFloat(ByteBuffer buffer) { - return (float) readDouble(buffer); - } - - @Override - public int readInt(ByteBuffer buffer) { - return (int) Math.round(readDouble(buffer)); - } - - @Override - public long readLong(ByteBuffer buffer) { - return (long) Math.round(readDouble(buffer)); - } - - @Override - public boolean hasNext(ByteBuffer buffer) throws IOException { - return (nextReadIndex < readTotalCount) || buffer.hasRemaining(); - } - - @Override - public void reset() { - nextReadIndex = 0; - readTotalCount = 0; - } - - private void loadBlock(ByteBuffer buffer) { - BitReader reader = new BitReader(buffer); - // Block size with 16 bits - this.readTotalCount = (int) reader.next(16); - // Number of reserved components with 16 bits - int m = (int) reader.next(16); - // Exponent of quantification level with 16 bits - int beta = (short) reader.next(16); - // Decode index sequence - int[] index = decodeIndex(m, reader); - // Decode value sequence - long[] value = decodeValue(m, reader); - reader.skip(); - // Quantification - double eps = Math.pow(2, beta); - this.data = new double[readTotalCount]; - for (int i = 0; i < m; i++) { - data[index[i]] = value[i] * eps; - } - DoubleDCT_1D dct = new DoubleDCT_1D(readTotalCount); - dct.inverse(data, true); - } - - private long[] decodeValue(int m, BitReader reader) { - if (m == 0) { - return new long[0]; - } - // Decode the encoded bit width of the first value with 8 bits - int bits = (int) reader.next(8); - // Decode min{|v|} - long min = reader.next(bits); - // Decode all values - long value[] = new long[m]; - int symbol; - for (int i = 0; i < m; i++) { - symbol = (int) reader.next(1); - value[i] = reader.next(bits); - bits = getValueWidth(value[i]); - value[i] += min; - if (symbol == 1) { // Negative value - value[i] = -value[i]; - } - } - return value; - } - - private int[] decodeIndex(int m, BitReader reader) { - int[] value = new int[m]; - int bitsWidth = getValueWidth(getValueWidth(readTotalCount - 1)); - for (int i = 0; i < m; i += 8) { - int bits = (int) reader.next(bitsWidth); - for (int j = i; j < Math.min(i + 8, m); j++) { - value[j] = (int) reader.next(bits); - } - } - return value; - } - - /** - * Get the valid bit width of x - * - * @param x - * @return valid bit width - */ - private int getValueWidth(long x) { - return 64 - Long.numberOfLeadingZeros(x); - } -} diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/FreqEncoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/FreqEncoder.java deleted file mode 100644 index dafa96384de..00000000000 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/FreqEncoder.java +++ /dev/null @@ -1,317 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.tsfile.encoding.encoder; - -import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; -import org.apache.iotdb.tsfile.utils.BitConstructor; - -import org.jtransforms.dct.DoubleDCT_1D; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.util.ArrayList; -import java.util.PriorityQueue; - -public class FreqEncoder extends Encoder { - - public static final String FREQ_ENCODING_SNR = "freq_encoding_snr"; - public static final String FREQ_ENCODING_BLOCK_SIZE = "freq_encoding_block_size"; - protected static final int BLOCK_DEFAULT_SIZE = 1024; - protected static final double DEFAULT_SNR = 40; - private static final Logger logger = LoggerFactory.getLogger(FreqEncoder.class); - private int blockSize; - protected int writeIndex = 0; - private double threshold = 1e-4; - private int beta; - private double[] dataBuffer; - private DoubleDCT_1D transformer; - - public FreqEncoder() { - this(BLOCK_DEFAULT_SIZE); - } - - public FreqEncoder(int size) { - this(size, DEFAULT_SNR); - } - - public FreqEncoder(int size, double snr) { - super(TSEncoding.FREQ); - this.blockSize = size; - this.transformer = new DoubleDCT_1D(blockSize); - this.dataBuffer = new double[blockSize]; - snr = Math.max(snr, 0); - this.threshold = Math.pow(10, -snr / 10); - } - - @Override - public void encode(double value, ByteArrayOutputStream out) { - dataBuffer[writeIndex] = value; - writeIndex++; - if (writeIndex == blockSize) { - flush(out); - } - } - - @Override - public void encode(float value, ByteArrayOutputStream out) { - encode((double) value, out); - } - - @Override - public void encode(int value, ByteArrayOutputStream out) { - encode((double) value, out); - } - - @Override - public void encode(long value, ByteArrayOutputStream out) { - encode((double) value, out); - } - - @Override - public void flush(ByteArrayOutputStream out) { - try { - flushBlock(out); - } catch (IOException e) { - logger.error("flush data to stream failed!", e); - } - } - - @Override - public int getOneItemMaxSize() { - return 13; - } - - @Override - public long getMaxByteSize() { - return 8 + 13 * writeIndex; - } - - private void flushBlock(ByteArrayOutputStream out) throws IOException { - if (writeIndex > 0) { - dct(); - ArrayList<Point> list = selectPoints(dataBuffer); - byte[] data = encodeBlock(list); - out.write(data); - writeIndex = 0; - } - } - - private void dct() { - DoubleDCT_1D dct = - (writeIndex == this.blockSize) ? this.transformer : new DoubleDCT_1D(writeIndex); - dct.forward(dataBuffer, true); - } - - private byte[] encodeBlock(ArrayList<Point> list) { - // Quantification - int m = list.size(); - int[] index = new int[m]; - long[] value = new long[m]; - double eps = Math.pow(2, beta); - for (int i = 0; i < m; i++) { - Point p = list.get(i); - index[i] = p.getIndex(); - value[i] = Math.round(p.getValue() / eps); - } - BitConstructor constructor = new BitConstructor(9 + 13 * m); - // Block size with 16 bits - constructor.add(writeIndex, 16); - // Number of reserved components with 16 bits - constructor.add(m, 16); - // Exponent of quantification level with 16 bits - constructor.add(beta, 16); - // Encode the index sequence - encodeIndex(index, constructor); - // Encode the value sequence - encodeValue(value, constructor); - constructor.pad(); - // return the encoded bytes - return constructor.toByteArray(); - } - - private void encodeIndex(int[] value, BitConstructor constructor) { - int bitsWidth = getValueWidth(getValueWidth(writeIndex - 1)); - for (int i = 0; i < value.length; i += 8) { - int bits = 0; - for (int j = i; j < Math.min(value.length, i + 8); j++) { - bits = Math.max(bits, getValueWidth(value[j])); - } - constructor.add(bits, bitsWidth); - for (int j = i; j < Math.min(value.length, i + 8); j++) { - constructor.add(value[j], bits); - } - } - } - - private void encodeValue(long[] value, BitConstructor constructor) { - if (value.length == 0) { - return; - } - // Encode the encoded bit width of the first value with 8 bits - int bits = getValueWidth(Math.abs(value[0])); - constructor.add(bits, 8); - // Encode min{|v|} - long min = Math.abs(value[value.length - 1]); - constructor.add(min, bits); - // Encode all values - for (int i = 0; i < value.length; i++) { - constructor.add(value[i] >= 0 ? 0 : 1, 1); // Symbol bit - value[i] = Math.abs(value[i]) - min; - constructor.add(value[i], bits); - bits = getValueWidth(value[i]); - } - } - - /** - * Get the valid bit width of x - * - * @param x - * @return valid bit width - */ - private int getValueWidth(long x) { - return 64 - Long.numberOfLeadingZeros(x); - } - - private int initBeta(double sum2) { - double temp = Math.sqrt(threshold * sum2 / (writeIndex * writeIndex)); - return (int) Math.max(max2Power(temp), Math.log(sum2) / (2 * Math.log(2)) - 60); - } - - /** - * Returns the exponent of the largest power of 2 that is less than or equal to x.<br> - * max{y|2^y &le x, y is an integer} - * - * @param x - * @return the exponent of the largest power of 2 that is less than or equal to x - */ - private int max2Power(double x) { - double ans = 1; - int exponent = 0; - if (x > 1) { - while (ans * 2 <= x) { - ans = ans * 2; - exponent++; - } - } else { - while (ans > x) { - ans = ans / 2; - exponent--; - } - } - return exponent; - } - - private ArrayList<Point> selectPoints(double a[]) { - // Keep the components with priority queue in the descending order of energy - double sum2 = 0; - Point point; - PriorityQueue<Point> queue = new PriorityQueue<>(writeIndex); - for (int i = 0; i < writeIndex; i++) { - point = new Point(i, a[i]); - queue.add(point); - sum2 += point.getPower(); - } - // Add components to keepList - this.beta = initBeta(sum2); - double systemError = sum2; - ArrayList<Point> keepList = new ArrayList<>(); - int m = 0; // Number of reserved components - double roundingError = 0; - double reduceBits = Double.MAX_VALUE; - boolean first = true; - do { - while (systemError + roundingError > threshold * sum2) { - point = queue.poll(); - if (point == null) { - systemError = 0; - break; - } - keepList.add(point); - systemError = systemError - point.getPower(); - roundingError = Math.pow(2, this.beta * 2) * keepList.size(); - } - double increaseBits = estimateIncreaseBits(keepList, m); - if (reduceBits <= increaseBits || systemError + roundingError > threshold * sum2) { - if (!first) { - keepList = new ArrayList(keepList.subList(0, m)); - this.beta--; - } - break; - } - // Increase quantification level - first = false; - m = keepList.size(); - reduceBits = m; - this.beta++; - roundingError = Math.pow(2, this.beta * 2) * m; - } while (true); - return keepList; - } - - /** - * Estimate the number of increased bits by reserving more components - * - * @param list The list of reserved components in this turn - * @param m The number of resereved components in last turn - * @return Estimated number of bits - */ - private double estimateIncreaseBits(ArrayList<Point> list, int m) { - double bits = 0; - double eps = Math.pow(2, beta); - for (int i = m; i < list.size(); i++) { - bits += getValueWidth(writeIndex - 1); // Index - bits += getValueWidth(Math.round(Math.abs(list.get(i).getValue()) / eps)); // Value - bits += 1; // Symbol - } - return bits; - } - - protected class Point implements Comparable<Point> { - - private final int index; - private final double value; - - public Point(int index, double value) { - this.index = index; - this.value = value; - } - - @Override - public int compareTo(Point o) { - return Double.compare(o.getPower(), this.getPower()); - } - - /** @return the index */ - public int getIndex() { - return index; - } - - /** @return the value */ - public double getValue() { - return value; - } - - public double getPower() { - return value * value; - } - } -} diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java index 035ab5ed764..91513335c6c 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java @@ -69,8 +69,6 @@ public abstract class TSEncodingBuilder { return new GorillaV2(); case DICTIONARY: return new Dictionary(); - case FREQ: - return new Freq(); case ZIGZAG: return new Zigzag(); case CHIMP: @@ -134,68 +132,6 @@ public abstract class TSEncodingBuilder { } } - /** for INT32, INT64, FLOAT, DOUBLE. */ - public static class Freq extends TSEncodingBuilder { - - private double snr = TSFileDescriptor.getInstance().getConfig().getFreqEncodingSNR(); - private int blockSize = TSFileDescriptor.getInstance().getConfig().getFreqEncodingBlockSize(); - - @Override - public Encoder getEncoder(TSDataType type) { - switch (type) { - case INT32: - case INT64: - case FLOAT: - case DOUBLE: - return new FreqEncoder(blockSize, snr); - default: - throw new UnSupportedDataTypeException("FREQ doesn't support data type: " + type); - } - } - - @Override - public void initFromProps(Map<String, String> props) { - // set SNR from initialized map or default value if not set - if (props == null || !props.containsKey(FreqEncoder.FREQ_ENCODING_SNR)) { - snr = TSFileDescriptor.getInstance().getConfig().getFreqEncodingSNR(); - } else { - try { - snr = Double.parseDouble(props.get(FreqEncoder.FREQ_ENCODING_SNR)); - } catch (NumberFormatException e) { - logger.warn( - "The format of FREQ encoding SNR {} is not correct." - + " Using default FREQ encoding SNR.", - props.get(FreqEncoder.FREQ_ENCODING_SNR)); - } - if (snr < 0) { - snr = TSFileDescriptor.getInstance().getConfig().getFreqEncodingSNR(); - logger.warn( - "cannot set FREQ encoding SNR to negative value, replaced with default value:{}", - snr); - } - } - // set block size from initialized map or default value if not set - if (props == null || !props.containsKey(FreqEncoder.FREQ_ENCODING_BLOCK_SIZE)) { - blockSize = TSFileDescriptor.getInstance().getConfig().getFreqEncodingBlockSize(); - } else { - try { - blockSize = Integer.parseInt(props.get(FreqEncoder.FREQ_ENCODING_BLOCK_SIZE)); - } catch (NumberFormatException e) { - logger.warn( - "The format of FREQ encoding block size {} is not correct." - + " Using default FREQ encoding block size.", - props.get(FreqEncoder.FREQ_ENCODING_BLOCK_SIZE)); - } - if (blockSize < 0) { - blockSize = TSFileDescriptor.getInstance().getConfig().getFreqEncodingBlockSize(); - logger.warn( - "cannot set FREQ encoding block size to negative value, replaced with default value:{}", - blockSize); - } - } - } - } - /** for ENUMS, INT32, BOOLEAN, INT64, FLOAT, DOUBLE. */ public static class Rle extends TSEncodingBuilder { diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/TSEncoding.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/TSEncoding.java index 8d83e7887ad..75eae76cb7c 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/TSEncoding.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/TSEncoding.java @@ -29,6 +29,7 @@ public enum TSEncoding { REGULAR((byte) 7), GORILLA((byte) 8), ZIGZAG((byte) 9), + @Deprecated FREQ((byte) 10), CHIMP((byte) 11), SPRINTZ((byte) 12), @@ -71,8 +72,6 @@ public enum TSEncoding { return TSEncoding.GORILLA; case 9: return TSEncoding.ZIGZAG; - case 10: - return TSEncoding.FREQ; case 11: return TSEncoding.CHIMP; case 12: diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BitConstructor.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BitConstructor.java deleted file mode 100644 index 7800fa98315..00000000000 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BitConstructor.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.tsfile.utils; - -public class BitConstructor { - - private static final int BITS_IN_A_BYTE = 8; - private static final long ALL_MASK = -1; - private final ByteArrayList data; - private byte cache = 0; - private int cnt = 0; - - public BitConstructor() { - this.data = new ByteArrayList(); - } - - public BitConstructor(int initialCapacity) { - this.data = new ByteArrayList(initialCapacity); - } - - public void add(long x, int len) { - x = x & ~(ALL_MASK << len); // Make sure that all bits expect the lowest len bits of x are 0 - while (len > 0) { - // Number of bits inserted into cache - int m = len + cnt >= BITS_IN_A_BYTE ? BITS_IN_A_BYTE - cnt : len; - len -= m; - cnt += m; - byte y = (byte) (x >> len); - y = (byte) (y << (BITS_IN_A_BYTE - cnt)); - cache = (byte) (cache | y); - x = x & ~(ALL_MASK << len); - if (cnt == BITS_IN_A_BYTE) { - pad(); - } - } - } - - public byte[] toByteArray() { - byte[] ret; - if (cnt > 0) { - data.add(cache); - ret = data.toArray(); - data.removeAtIndex(data.size() - 1); - } else { - ret = data.toArray(); - } - return ret; - } - - public void clear() { - data.clear(); - cache = 0x00; - cnt = 0; - } - - /** Fill the rest part of cache with 0 */ - public void pad() { - if (cnt > 0) { - data.add(cache); - cache = 0x00; - cnt = 0; - } - } - - public void add(byte[] bytes) { - if (cnt == 0) { - data.addAll(bytes); - } else { - for (byte aByte : bytes) { - add(aByte, 8); - } - } - } - - public int sizeInBytes() { - return data.size() + (cnt > 0 ? 1 : 0); - } -} diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BitReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BitReader.java deleted file mode 100644 index 94926ebe1b7..00000000000 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BitReader.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright 2021 The Apache Software Foundation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.iotdb.tsfile.utils; - -import java.nio.ByteBuffer; - -public class BitReader { - - private static final int BITS_IN_A_BYTE = 8; - private static final byte MASKS[] = {(byte) 0xff, 0x7f, 0x3f, 0x1f, 0x0f, 0x07, 0x03, 0x01}; - private final ByteBuffer buffer; - private int bitCnt = BITS_IN_A_BYTE; - private byte cache = 0; - - public BitReader(ByteBuffer buffer) { - this.buffer = buffer; - } - - public long next(int len) { - long ret = 0; - while (len > 0) { - if (bitCnt == BITS_IN_A_BYTE) { - next(); - } - // Number of bits read from the current byte - int m = len + bitCnt >= BITS_IN_A_BYTE ? BITS_IN_A_BYTE - bitCnt : len; - len -= m; - ret = ret << m; - byte y = (byte) (cache & MASKS[bitCnt]); // Truncate the low bits with & - y = (byte) ((y & 0xff) >>> (BITS_IN_A_BYTE - bitCnt - m)); // Logical shift right - ret = ret | (y & 0xff); - bitCnt += m; - } - return ret; - } - - public byte[] nextBytes(int len) { - byte[] ret = new byte[len]; - if (bitCnt == BITS_IN_A_BYTE) { - buffer.get(ret); - } else { - for (int i = 0; i < len; i++) { - ret[i] = (byte) next(8); - } - } - return ret; - } - - public void skip() { - this.bitCnt = BITS_IN_A_BYTE; - } - - private void next() { - this.cache = buffer.get(); - this.bitCnt = 0; - } -} diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/FreqDecoderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/FreqDecoderTest.java deleted file mode 100644 index 1f72b43536e..00000000000 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/FreqDecoderTest.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.tsfile.encoding.decoder; - -import org.apache.iotdb.tsfile.encoding.encoder.FreqEncoder; - -import org.jtransforms.fft.DoubleFFT_1D; -import org.junit.Before; -import org.junit.Test; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Random; - -import static org.junit.Assert.assertTrue; - -public class FreqDecoderTest { - - private static final int ROW_NUM = 1024; - ByteArrayOutputStream out; - private FreqEncoder writer; - private FreqDecoder reader; - private final Random ran = new Random(); - private ByteBuffer buffer; - - @Before - public void test() { - writer = new FreqEncoder(); - reader = new FreqDecoder(); - } - - @Test - public void testSin() throws IOException { - reader.reset(); - double[] data = new double[ROW_NUM]; - for (int i = 0; i < ROW_NUM; i++) { - data[i] = Math.cos(0.25 * Math.PI * i); - } - double[] recover = shouldReadAndWrite(data, ROW_NUM); - assertTrue(SNR(data, recover, ROW_NUM) > 40); - } - - @Test - public void testBoundInt() throws IOException { - reader.reset(); - double[] data = new double[ROW_NUM]; - for (int i = 2; i < 21; i++) { - boundInt(i, data); - } - } - - @Test - public void testLongTail() throws IOException { - reader.reset(); - double[] a = new double[ROW_NUM * 2]; - for (int i = 0; i < ROW_NUM; i++) { - double amp = Math.exp(-i); - double theta = ran.nextDouble() * 2 * Math.PI; - a[i * 2] = amp * Math.cos(theta); - a[i * 2 + 1] = amp * Math.sin(theta); - } - DoubleFFT_1D fft = new DoubleFFT_1D(ROW_NUM); - fft.complexInverse(a, false); - double data[] = new double[ROW_NUM]; - for (int i = 0; i < ROW_NUM; i++) { - data[i] = a[i * 2]; - } - double[] recover = shouldReadAndWrite(data, ROW_NUM); - assertTrue(SNR(data, recover, ROW_NUM) > 40); - } - - @Test - public void testMaxMin() throws IOException { - reader.reset(); - double[] data = new double[ROW_NUM]; - for (int i = 0; i < ROW_NUM; i++) { - data[i] = (i & 1) == 0 ? Long.MAX_VALUE : Long.MIN_VALUE; - } - double[] recover = shouldReadAndWrite(data, ROW_NUM); - assertTrue(SNR(data, recover, ROW_NUM) > 40); - } - - @Test - public void testConstantInt() throws IOException { - reader.reset(); - double[] data = new double[ROW_NUM]; - for (int i = 0; i < 10; i++) { - constantInt(i, data); - } - } - - private void boundInt(int power, double[] data) throws IOException { - reader.reset(); - double maxn = 1 << power; - for (int i = 0; i < ROW_NUM; i++) { - data[i] = ran.nextDouble() * maxn; - } - double[] recover = shouldReadAndWrite(data, ROW_NUM); - assertTrue(SNR(data, recover, ROW_NUM) > 40); - } - - private void constantInt(int value, double[] data) throws IOException { - reader.reset(); - for (int i = 0; i < ROW_NUM; i++) { - data[i] = value; - } - double[] recover = shouldReadAndWrite(data, ROW_NUM); - assertTrue(SNR(data, recover, ROW_NUM) > 40); - } - - private void writeData(double[] data, int length) { - for (int i = 0; i < length; i++) { - writer.encode(data[i], out); - } - writer.flush(out); - } - - private double[] shouldReadAndWrite(double[] data, int length) throws IOException { - out = new ByteArrayOutputStream(); - writeData(data, length); - byte[] page = out.toByteArray(); - buffer = ByteBuffer.wrap(page); - int i = 0; - double recover[] = new double[length]; - while (reader.hasNext(buffer)) { - recover[i] = reader.readDouble(buffer); - i++; - } - return recover; - } - - public double SNR(double[] gd, double[] x, int length) { - double noise_power = 0, signal_power = 0; - for (int i = 0; i < length; i++) { - noise_power += (gd[i] - x[i]) * (gd[i] - x[i]); - signal_power += gd[i] * gd[i]; - } - if (noise_power == 0) { - return Double.POSITIVE_INFINITY; - } else { - return 10 * Math.log10(signal_power / noise_power); - } - } -}
