This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new dc38c7a [IOTDB-2609] A new lossy encoding method based on frequency
domain (#5118)
dc38c7a is described below
commit dc38c7adc183f854eb52010bd09180badc6a708b
Author: Haoyu Wang <[email protected]>
AuthorDate: Fri Mar 18 10:34:20 2022 +0800
[IOTDB-2609] A new lossy encoding method based on frequency domain (#5118)
Co-authored-by: Haonan <[email protected]>
---
.../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 7 +-
client-cpp/src/main/Session.h | 3 +-
client-py/iotdb/utils/IoTDBConstants.py | 1 +
docs/UserGuide/Data-Concept/Encoding.md | 14 +-
docs/UserGuide/Reference/Config-Manual.md | 18 ++
docs/zh/UserGuide/Data-Concept/Encoding.md | 14 +-
docs/zh/UserGuide/Reference/Config-Manual.md | 20 ++
.../util/TSFileConfigUtilCompletenessTest.java | 4 +-
.../iotdb/db/integration/IoTDBEncodingIT.java | 76 +++++
.../resources/conf/iotdb-engine.properties | 8 +
.../org/apache/iotdb/db/utils/SchemaUtils.java | 1 +
tsfile/pom.xml | 14 +-
.../iotdb/tsfile/common/conf/TSFileConfig.java | 20 ++
.../iotdb/tsfile/common/conf/TSFileDescriptor.java | 6 +
.../iotdb/tsfile/encoding/decoder/Decoder.java | 3 +-
.../iotdb/tsfile/encoding/decoder/FreqDecoder.java | 140 +++++++++
.../iotdb/tsfile/encoding/encoder/FreqEncoder.java | 313 +++++++++++++++++++++
.../tsfile/encoding/encoder/TSEncodingBuilder.java | 65 ++++-
.../tsfile/file/metadata/enums/TSEncoding.java | 5 +-
.../apache/iotdb/tsfile/utils/BitConstructor.java | 93 ++++++
.../org/apache/iotdb/tsfile/utils/BitReader.java | 70 +++++
.../tsfile/encoding/decoder/FreqDecoderTest.java | 161 +++++++++++
22 files changed, 1040 insertions(+), 16 deletions(-)
diff --git a/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4
b/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4
index fd34160..07ed512 100644
--- a/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4
+++ b/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4
@@ -629,7 +629,7 @@ TEXT
// Encoding Type Keywords
ENCODING_VALUE
- : DICTIONARY | DIFF | GORILLA | PLAIN | REGULAR | RLE | TS_2DIFF | ZIGZAG
+ : DICTIONARY | DIFF | GORILLA | PLAIN | REGULAR | RLE | TS_2DIFF | ZIGZAG
| FREQ
;
DICTIONARY
@@ -664,6 +664,11 @@ ZIGZAG
: Z I G Z A G
;
+FREQ
+ : F R E Q
+ ;
+
+
// Compressor Type Keywords
COMPRESSOR_VALUE
diff --git a/client-cpp/src/main/Session.h b/client-cpp/src/main/Session.h
index c8b8326..b3eea5d 100644
--- a/client-cpp/src/main/Session.h
+++ b/client-cpp/src/main/Session.h
@@ -144,7 +144,8 @@ namespace TSEncoding {
GORILLA_V1 = (char) 6,
REGULAR = (char) 7,
GORILLA = (char) 8,
- ZIGZAG = (char) 9
+ ZIGZAG = (char) 9,
+ FREQ = (char) 10
};
}
diff --git a/client-py/iotdb/utils/IoTDBConstants.py
b/client-py/iotdb/utils/IoTDBConstants.py
index b55af5f..7b992e3 100644
--- a/client-py/iotdb/utils/IoTDBConstants.py
+++ b/client-py/iotdb/utils/IoTDBConstants.py
@@ -49,6 +49,7 @@ class TSEncoding(Enum):
REGULAR = 7
GORILLA = 8
ZIGZAG = 9
+ FREQ = 10
# this method is implemented to avoid the issue reported by:
# https://bugs.python.org/issue30545
diff --git a/docs/UserGuide/Data-Concept/Encoding.md
b/docs/UserGuide/Data-Concept/Encoding.md
index bd8fe21..11f2c91 100644
--- a/docs/UserGuide/Data-Concept/Encoding.md
+++ b/docs/UserGuide/Data-Concept/Encoding.md
@@ -54,6 +54,12 @@ Usage restrictions: When using GORILLA to encode INT32 data,
you need to ensure
DICTIONARY encoding is lossless. It is suitable for TEXT data with low
cardinality (i.e. low number of distinct values). It is not recommended to use
it for high-cardinality data.
+* FREQ
+
+FREQ encoding is lossy. It transforms the time sequence to the frequency
domain and only reserve part of the frequency components with high energy. It
is more suitable for sequence with obvious periodicity.
+
+> There are two parameters of FREQ encoding in the configuration file:
`freq_snr` defines the signal-noise-ratio (SNR). Both the compression ratio and
accuracy loss decrease when it increases. `freq_block_size` defines the data
size in a time-frequency transformation. It is not recommended to modify the
default value. The detailed experimental results and analysis of the influences
of parameters are in the design document.
+
* ZIGZAG
ZIGZAG encoding maps signed integers to unsigned integers so that numbers with
a small absolute value (for instance, -1) have a small variant encoded value
too. It does this in a way that "zig-zags" back and forth through the positive
and negative integers.
@@ -69,10 +75,10 @@ The five encodings described in the previous sections are
applicable to differen
|Data Type |Supported Encoding|
|:---:|:---:|
|BOOLEAN| PLAIN, RLE|
-|INT32 |PLAIN, RLE, TS_2DIFF, GORILLA, ZIGZAG|
-|INT64 |PLAIN, RLE, TS_2DIFF, GORILLA, ZIGZAG|
-|FLOAT |PLAIN, RLE, TS_2DIFF, GORILLA|
-|DOUBLE |PLAIN, RLE, TS_2DIFF, GORILLA|
+|INT32 |PLAIN, RLE, TS_2DIFF, GORILLA, FREQ, ZIGZAG|
+|INT64 |PLAIN, RLE, TS_2DIFF, GORILLA, FREQ, ZIGZAG|
+|FLOAT |PLAIN, RLE, TS_2DIFF, GORILLA, FREQ|
+|DOUBLE |PLAIN, RLE, TS_2DIFF, GORILLA, FREQ|
|TEXT |PLAIN, DICTIONARY|
</div>
diff --git a/docs/UserGuide/Reference/Config-Manual.md
b/docs/UserGuide/Reference/Config-Manual.md
index fece0eb..b08da16 100644
--- a/docs/UserGuide/Reference/Config-Manual.md
+++ b/docs/UserGuide/Reference/Config-Manual.md
@@ -197,6 +197,24 @@ The permission definitions are in
${IOTDB\_CONF}/conf/jmx.access.
|Effective|After restarting system|
+* freq_snr
+
+|Name| freq_snr |
+|:---:|:---|
+|Description| Signal-noise-ratio (SNR) of lossy FREQ encoding |
+|Type|Double|
+|Default| 40.0 |
+|Effective|Trigger|
+
+
+* freq_block_size
+
+|Name| freq_block_size |
+|:---:|:---|
+|Description| Block size of FREQ encoding. In other words, the number of data
points in a time-frequency transformation. To speed up the encoding, it is
recommended to be the power of 2. |
+|Type|Int32|
+|Default| 1024 |
+|Effective|Trigger|
### Engine Layer
diff --git a/docs/zh/UserGuide/Data-Concept/Encoding.md
b/docs/zh/UserGuide/Data-Concept/Encoding.md
index bebd17e..7671971 100644
--- a/docs/zh/UserGuide/Data-Concept/Encoding.md
+++ b/docs/zh/UserGuide/Data-Concept/Encoding.md
@@ -53,6 +53,12 @@ GORILLA 编码是一种无损编码，它比较适合编码前后值比较接近
字典编码是一种无损编码。它适合编码基数小的数据（即数据去重后唯一值数量小）。不推荐用于基数大的数据。
+* 频域编码 （FREQ）
+
+频域编码是一种有损编码，它将时序数据变换为频域，仅保留部分高能量的频域分量。该编码适合于具有明显周期性的数据。
+
+>
频域编码在配置文件中包括两个参数：`freq_snr`指定了编码的信噪比，该参数增大会同时降低压缩比和精度损失；`freq_block_size`指定了编码进行时频域变换的分组大小，推荐不对默认值进行修改。参数影响的实验结果和分析详见设计文档。
+
* ZIGZAG 编码
ZigZag编码将有符号整型映射到无符号整型，适合比较小的整数。
@@ -68,10 +74,10 @@ ZigZag编码将有符号整型映射到无符号整型，适合比较小的整
|数据类型 |支持的编码|
|:---:|:---:|
|BOOLEAN| PLAIN, RLE|
-|INT32 |PLAIN, RLE, TS_2DIFF, GORILLA, ZIGZAG|
-|INT64 |PLAIN, RLE, TS_2DIFF, GORILLA, ZIGZAG|
-|FLOAT |PLAIN, RLE, TS_2DIFF, GORILLA|
-|DOUBLE |PLAIN, RLE, TS_2DIFF, GORILLA|
+|INT32 |PLAIN, RLE, TS_2DIFF, GORILLA, FREQ, ZIGZAG|
+|INT64 |PLAIN, RLE, TS_2DIFF, GORILLA, FREQ, ZIGZAG|
+|FLOAT |PLAIN, RLE, TS_2DIFF, GORILLA, FREQ|
+|DOUBLE |PLAIN, RLE, TS_2DIFF, GORILLA, FREQ|
|TEXT |PLAIN, DICTIONARY|
</div>
diff --git a/docs/zh/UserGuide/Reference/Config-Manual.md
b/docs/zh/UserGuide/Reference/Config-Manual.md
index 2f9e76d..ef00a49 100644
--- a/docs/zh/UserGuide/Reference/Config-Manual.md
+++ b/docs/zh/UserGuide/Reference/Config-Manual.md
@@ -210,6 +210,26 @@ Server，客户端的使用方式详见 [SQL 命令行终端（CLI）](https://i
|默认值| 100 |
|改后生效方式|触发生效|
+
+* freq_snr
+
+|名字| freq_snr |
+|:---:|:---|
+|描述| 有损的FREQ编码的信噪比 |
+|类型|Double|
+|默认值| 40.0 |
+|改后生效方式|触发生效|
+
+
+* freq_block_size
+
+|名字| freq_block_size |
+|:---:|:---|
+|描述| FREQ编码的块大小，即一次时频域变换的数据点个数。为了加快编码速度，建议将其设置为2的幂次。 |
+|类型|Int32|
+|默认值| 1024 |
+|改后生效方式|触发生效|
+
### 目录配置
* system\_dir
diff --git
a/flink-tsfile-connector/src/test/java/org/apache/iotdb/flink/util/TSFileConfigUtilCompletenessTest.java
b/flink-tsfile-connector/src/test/java/org/apache/iotdb/flink/util/TSFileConfigUtilCompletenessTest.java
index 0c51187..156baa4 100644
---
a/flink-tsfile-connector/src/test/java/org/apache/iotdb/flink/util/TSFileConfigUtilCompletenessTest.java
+++
b/flink-tsfile-connector/src/test/java/org/apache/iotdb/flink/util/TSFileConfigUtilCompletenessTest.java
@@ -70,7 +70,9 @@ public class TSFileConfigUtilCompletenessTest {
"setTimeSeriesDataType",
"setTSFileStorageFs",
"setUseKerberos",
- "setValueEncoder"
+ "setValueEncoder",
+ "setFreqEncodingSNR",
+ "setFreqEncodingBlockSize"
};
Set<String> newSetters =
Arrays.stream(TSFileConfig.class.getMethods())
diff --git
a/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBEncodingIT.java
b/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBEncodingIT.java
index c4a7bdc..2782d20 100644
---
a/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBEncodingIT.java
+++
b/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBEncodingIT.java
@@ -40,6 +40,7 @@ import java.util.Arrays;
import java.util.List;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@Category({LocalStandaloneTest.class})
@@ -309,6 +310,68 @@ public class IoTDBEncodingIT {
}
@Test
+ public void testSetTimeEncoderRegularAndValueEncoderFREQ() {
+ try (Connection connection =
+ DriverManager.getConnection(
+ Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+ statement.execute(
+ "CREATE TIMESERIES root.db_0.tab0.salary WITH
DATATYPE=INT64,ENCODING=FREQ");
+ statement.execute("insert into root.db_0.tab0(time,salary)
values(1,1100)");
+ statement.execute("insert into root.db_0.tab0(time,salary)
values(2,1200)");
+ statement.execute("insert into root.db_0.tab0(time,salary)
values(3,1300)");
+ statement.execute("insert into root.db_0.tab0(time,salary)
values(4,1400)");
+ statement.execute("flush");
+
+ int[] groundtruth = new int[] {1100, 1200, 1300, 1400};
+ int[] result = new int[4];
+ try (ResultSet resultSet = statement.executeQuery("select * from
root.db_0.tab0")) {
+ int index = 0;
+ while (resultSet.next()) {
+ int salary = resultSet.getInt("root.db_0.tab0.salary");
+ result[index] = salary;
+ index++;
+ }
+ assertTrue(SNR(groundtruth, result, groundtruth.length) > 40);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail();
+ }
+ }
+
+ @Test
+ public void testSetTimeEncoderRegularAndValueEncoderFREQOutofOrder() {
+ try (Connection connection =
+ DriverManager.getConnection(
+ Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+ statement.execute(
+ "CREATE TIMESERIES root.db_0.tab0.salary WITH
DATATYPE=INT64,ENCODING=FREQ");
+ statement.execute("insert into root.db_0.tab0(time,salary)
values(1,1200)");
+ statement.execute("insert into root.db_0.tab0(time,salary)
values(2,1100)");
+ statement.execute("insert into root.db_0.tab0(time,salary)
values(7,1000)");
+ statement.execute("insert into root.db_0.tab0(time,salary)
values(4,2200)");
+ statement.execute("flush");
+
+ int[] groundtruth = new int[] {1200, 1100, 2200, 1000};
+ int[] result = new int[4];
+ try (ResultSet resultSet = statement.executeQuery("select * from
root.db_0.tab0")) {
+ int index = 0;
+ while (resultSet.next()) {
+ int salary = resultSet.getInt("root.db_0.tab0.salary");
+ result[index] = salary;
+ index++;
+ }
+ assertTrue(SNR(groundtruth, result, groundtruth.length) > 40);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail();
+ }
+ }
+
+ @Test
public void testSetTimeEncoderRegularAndValueEncoderDictionary() {
try (Connection connection =
DriverManager.getConnection(
@@ -413,4 +476,17 @@ public class IoTDBEncodingIT {
e.printStackTrace();
}
}
+
+ public double SNR(int[] gd, int[] x, int length) {
+ double noise_power = 0, signal_power = 0;
+ for (int i = 0; i < length; i++) {
+ noise_power += (gd[i] - x[i]) * (gd[i] - x[i]);
+ signal_power += gd[i] * gd[i];
+ }
+ if (noise_power == 0) {
+ return Double.POSITIVE_INFINITY;
+ } else {
+ return 10 * Math.log10(signal_power / noise_power);
+ }
+ }
}
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties
b/server/src/assembly/resources/conf/iotdb-engine.properties
index fd508eb..3d0ea67 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -704,6 +704,14 @@ timestamp_precision=ms
# Datatype: long
# slow_query_threshold=5000
+# Signal-noise-ratio (SNR) of FREQ encoding
+# Datatype: double
+# freq_snr=40.0
+
+# Block size of FREQ encoding
+# Datatype: integer
+# freq_block_size=1024
+
####################
### MQTT Broker Configuration
####################
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java
b/server/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java
index 3c6e332..eee8458 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java
@@ -69,6 +69,7 @@ public class SchemaUtils {
intSet.add(TSEncoding.TS_2DIFF);
intSet.add(TSEncoding.GORILLA);
intSet.add(TSEncoding.ZIGZAG);
+ intSet.add(TSEncoding.FREQ);
schemaChecker.put(TSDataType.INT32, intSet);
schemaChecker.put(TSDataType.INT64, intSet);
diff --git a/tsfile/pom.xml b/tsfile/pom.xml
index cb86d42..beb2149 100644
--- a/tsfile/pom.xml
+++ b/tsfile/pom.xml
@@ -55,6 +55,16 @@
<version>1.8.0</version>
</dependency>
<dependency>
+ <groupId>com.github.wendykierp</groupId>
+ <artifactId>JTransforms</artifactId>
+ <version>3.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.collections</groupId>
+ <artifactId>eclipse-collections</artifactId>
+ <version>10.4.0</version>
+ </dependency>
+ <dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
</dependency>
@@ -62,8 +72,8 @@
<build>
<plugins>
<!--
- Generate an OSGI compatible MANIFEST file.
- -->
+ Generate an OSGI compatible MANIFEST file.
+ -->
<plugin>
<groupId>org.apache.felix</groupId>
<artifactId>maven-bundle-plugin</artifactId>
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
index 060aba9..bc39353 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
@@ -107,6 +107,10 @@ public class TSFileConfig implements Serializable {
private double sdtMaxError = 100;
/** Default DFT satisfy rate is 0.1 */
private double dftSatisfyRate = 0.1;
+ /** Default SNR for FREQ encoding is 40dB. */
+ private double freqEncodingSNR = 40;
+ /** Default block size for FREQ encoding is 1024. */
+ private int freqEncodingBlockSize = 1024;
/** Data compression method, TsFile supports UNCOMPRESSED, SNAPPY or LZ4. */
private CompressionType compressor = CompressionType.SNAPPY;
/** Line count threshold for checking page memory occupied size. */
@@ -411,4 +415,20 @@ public class TSFileConfig implements Serializable {
public void setBatchSize(int batchSize) {
this.batchSize = batchSize;
}
+
+ public double getFreqEncodingSNR() {
+ return freqEncodingSNR;
+ }
+
+ public void setFreqEncodingSNR(double freqEncodingSNR) {
+ this.freqEncodingSNR = freqEncodingSNR;
+ }
+
+ public int getFreqEncodingBlockSize() {
+ return freqEncodingBlockSize;
+ }
+
+ public void setFreqEncodingBlockSize(int freqEncodingBlockSize) {
+ this.freqEncodingBlockSize = freqEncodingBlockSize;
+ }
}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java
index 840e64d..965a12e 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java
@@ -81,6 +81,8 @@ public class TSFileDescriptor {
writer.setString(conf::setValueEncoder, "value_encoder");
writer.setString(conf::setCompressor, "compressor");
writer.setInt(conf::setBatchSize, "batch_size");
+ writer.setInt(conf::setFreqEncodingBlockSize, "freq_block_size");
+ writer.setDouble(conf::setFreqEncodingSNR, "freq_snr");
}
private class PropertiesOverWriter {
@@ -98,6 +100,10 @@ public class TSFileDescriptor {
set(setter, propertyKey, Integer::parseInt);
}
+ public void setDouble(Consumer<Double> setter, String propertyKey) {
+ set(setter, propertyKey, Double::parseDouble);
+ }
+
public void setString(Consumer<String> setter, String propertyKey) {
set(setter, propertyKey, Function.identity());
}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/Decoder.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/Decoder.java
index e9ce5b5..d0cf2f3 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/Decoder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/Decoder.java
@@ -121,7 +121,8 @@ public abstract class Decoder {
default:
throw new TsFileDecodingException(String.format(ERROR_MSG,
encoding, dataType));
}
-
+ case FREQ:
+ return new FreqDecoder();
default:
throw new TsFileDecodingException(String.format(ERROR_MSG, encoding,
dataType));
}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/FreqDecoder.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/FreqDecoder.java
new file mode 100644
index 0000000..3797681
--- /dev/null
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/FreqDecoder.java
@@ -0,0 +1,140 @@
+/*
+ * Copyright 2021 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.iotdb.tsfile.encoding.decoder;
+
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.utils.BitReader;
+
+import org.jtransforms.dct.DoubleDCT_1D;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+public class FreqDecoder extends Decoder {
+
+ private double data[];
+
+ private int readTotalCount = 0;
+
+ private int nextReadIndex = 0;
+
+ public FreqDecoder() {
+ super(TSEncoding.FREQ);
+ }
+
+ @Override
+ public double readDouble(ByteBuffer buffer) {
+ if (nextReadIndex == readTotalCount) {
+ loadBlock(buffer);
+ nextReadIndex = 0;
+ }
+ return data[nextReadIndex++];
+ }
+
+ @Override
+ public float readFloat(ByteBuffer buffer) {
+ return (float) readDouble(buffer);
+ }
+
+ @Override
+ public int readInt(ByteBuffer buffer) {
+ return (int) Math.round(readDouble(buffer));
+ }
+
+ @Override
+ public long readLong(ByteBuffer buffer) {
+ return (long) Math.round(readDouble(buffer));
+ }
+
+ @Override
+ public boolean hasNext(ByteBuffer buffer) throws IOException {
+ return (nextReadIndex < readTotalCount) || buffer.hasRemaining();
+ }
+
+ @Override
+ public void reset() {
+ nextReadIndex = 0;
+ readTotalCount = 0;
+ }
+
+ private void loadBlock(ByteBuffer buffer) {
+ BitReader reader = new BitReader(buffer);
+ // Block size with 16 bits
+ this.readTotalCount = (int) reader.next(16);
+ // Number of reserved components with 16 bits
+ int m = (int) reader.next(16);
+ // Exponent of quantification level with 16 bits
+ int beta = (short) reader.next(16);
+ // Decode index sequence
+ int[] index = decodeIndex(m, reader);
+ // Decode value sequence
+ long[] value = decodeValue(m, reader);
+ reader.skip();
+ // Quantification
+ double eps = Math.pow(2, beta);
+ this.data = new double[readTotalCount];
+ for (int i = 0; i < m; i++) {
+ data[index[i]] = value[i] * eps;
+ }
+ DoubleDCT_1D dct = new DoubleDCT_1D(readTotalCount);
+ dct.inverse(data, true);
+ }
+
+ private long[] decodeValue(int m, BitReader reader) {
+ if (m == 0) {
+ return new long[0];
+ }
+ // Decode the encoded bit width of the first value with 8 bits
+ int bits = (int) reader.next(8);
+ // Decode min{|v|}
+ long min = reader.next(bits);
+ // Decode all values
+ long value[] = new long[m];
+ int symbol;
+ for (int i = 0; i < m; i++) {
+ symbol = (int) reader.next(1);
+ value[i] = reader.next(bits);
+ bits = getValueWidth(value[i]);
+ value[i] += min;
+ if (symbol == 1) { // Negative value
+ value[i] = -value[i];
+ }
+ }
+ return value;
+ }
+
+ private int[] decodeIndex(int m, BitReader reader) {
+ int[] value = new int[m];
+ int bitsWidth = getValueWidth(getValueWidth(readTotalCount - 1));
+ for (int i = 0; i < m; i += 8) {
+ int bits = (int) reader.next(bitsWidth);
+ for (int j = i; j < Math.min(i + 8, m); j++) {
+ value[j] = (int) reader.next(bits);
+ }
+ }
+ return value;
+ }
+
+ /**
+ * Get the valid bit width of x
+ *
+ * @param x
+ * @return valid bit width
+ */
+ private int getValueWidth(long x) {
+ return 64 - Long.numberOfLeadingZeros(x);
+ }
+}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/FreqEncoder.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/FreqEncoder.java
new file mode 100644
index 0000000..39e5c4e
--- /dev/null
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/FreqEncoder.java
@@ -0,0 +1,313 @@
+/*
+ * Copyright 2021 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.iotdb.tsfile.encoding.encoder;
+
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.utils.BitConstructor;
+
+import org.jtransforms.dct.DoubleDCT_1D;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.PriorityQueue;
+
+public class FreqEncoder extends Encoder {
+
+ public static final String FREQ_ENCODING_SNR = "freq_encoding_snr";
+ public static final String FREQ_ENCODING_BLOCK_SIZE =
"freq_encoding_block_size";
+ protected static final int BLOCK_DEFAULT_SIZE = 1024;
+ protected static final double DEFAULT_SNR = 40;
+ private static final Logger logger =
LoggerFactory.getLogger(FreqEncoder.class);
+ private int blockSize;
+ protected int writeIndex = 0;
+ private double threshold = 1e-4;
+ private int beta;
+ private double[] dataBuffer;
+ private DoubleDCT_1D transformer;
+
+ public FreqEncoder() {
+ this(BLOCK_DEFAULT_SIZE);
+ }
+
+ public FreqEncoder(int size) {
+ this(size, DEFAULT_SNR);
+ }
+
+ public FreqEncoder(int size, double snr) {
+ super(TSEncoding.FREQ);
+ this.blockSize = size;
+ this.transformer = new DoubleDCT_1D(blockSize);
+ this.dataBuffer = new double[blockSize];
+ snr = Math.max(snr, 0);
+ this.threshold = Math.pow(10, -snr / 10);
+ }
+
+ @Override
+ public void encode(double value, ByteArrayOutputStream out) {
+ dataBuffer[writeIndex] = value;
+ writeIndex++;
+ if (writeIndex == blockSize) {
+ flush(out);
+ }
+ }
+
+ @Override
+ public void encode(float value, ByteArrayOutputStream out) {
+ encode((double) value, out);
+ }
+
+ @Override
+ public void encode(int value, ByteArrayOutputStream out) {
+ encode((double) value, out);
+ }
+
+ @Override
+ public void encode(long value, ByteArrayOutputStream out) {
+ encode((double) value, out);
+ }
+
+ @Override
+ public void flush(ByteArrayOutputStream out) {
+ try {
+ flushBlock(out);
+ } catch (IOException e) {
+ logger.error("flush data to stream failed!", e);
+ }
+ }
+
+ @Override
+ public int getOneItemMaxSize() {
+ return 13;
+ }
+
+ @Override
+ public long getMaxByteSize() {
+ return 8 + 13 * writeIndex;
+ }
+
+ private void flushBlock(ByteArrayOutputStream out) throws IOException {
+ if (writeIndex > 0) {
+ dct();
+ ArrayList<Point> list = selectPoints(dataBuffer);
+ byte[] data = encodeBlock(list);
+ out.write(data);
+ writeIndex = 0;
+ }
+ }
+
+ private void dct() {
+ DoubleDCT_1D dct =
+ (writeIndex == this.blockSize) ? this.transformer : new
DoubleDCT_1D(writeIndex);
+ dct.forward(dataBuffer, true);
+ }
+
+ private byte[] encodeBlock(ArrayList<Point> list) {
+ // Quantification
+ int m = list.size();
+ int[] index = new int[m];
+ long[] value = new long[m];
+ double eps = Math.pow(2, beta);
+ for (int i = 0; i < m; i++) {
+ Point p = list.get(i);
+ index[i] = p.getIndex();
+ value[i] = Math.round(p.getValue() / eps);
+ }
+ BitConstructor constructor = new BitConstructor(9 + 13 * m);
+ // Block size with 16 bits
+ constructor.add(writeIndex, 16);
+ // Number of reserved components with 16 bits
+ constructor.add(m, 16);
+ // Exponent of quantification level with 16 bits
+ constructor.add(beta, 16);
+ // Encode the index sequence
+ encodeIndex(index, constructor);
+ // Encode the value sequence
+ encodeValue(value, constructor);
+ constructor.pad();
+ // return the encoded bytes
+ return constructor.toByteArray();
+ }
+
+ private void encodeIndex(int[] value, BitConstructor constructor) {
+ int bitsWidth = getValueWidth(getValueWidth(writeIndex - 1));
+ for (int i = 0; i < value.length; i += 8) {
+ int bits = 0;
+ for (int j = i; j < Math.min(value.length, i + 8); j++) {
+ bits = Math.max(bits, getValueWidth(value[j]));
+ }
+ constructor.add(bits, bitsWidth);
+ for (int j = i; j < Math.min(value.length, i + 8); j++) {
+ constructor.add(value[j], bits);
+ }
+ }
+ }
+
+ private void encodeValue(long[] value, BitConstructor constructor) {
+ if (value.length == 0) {
+ return;
+ }
+ // Encode the encoded bit width of the first value with 8 bits
+ int bits = getValueWidth(Math.abs(value[0]));
+ constructor.add(bits, 8);
+ // Encode min{|v|}
+ long min = Math.abs(value[value.length - 1]);
+ constructor.add(min, bits);
+ // Encode all values
+ for (int i = 0; i < value.length; i++) {
+ constructor.add(value[i] >= 0 ? 0 : 1, 1); // Symbol bit
+ value[i] = Math.abs(value[i]) - min;
+ constructor.add(value[i], bits);
+ bits = getValueWidth(value[i]);
+ }
+ }
+
+ /**
+ * Get the valid bit width of x
+ *
+ * @param x
+ * @return valid bit width
+ */
+ private int getValueWidth(long x) {
+ return 64 - Long.numberOfLeadingZeros(x);
+ }
+
+ private int initBeta(double sum2) {
+ double temp = Math.sqrt(threshold * sum2 / (writeIndex * writeIndex));
+ return (int) Math.max(max2Power(temp), Math.log(sum2) / (2 * Math.log(2))
- 60);
+ }
+
+ /**
+ * Returns the exponent of the largest power of 2 that is less than or equal
to x.<br>
+ * max{y|2^y &le x, y is an integer}
+ *
+ * @param x
+ * @return the exponent of the largest power of 2 that is less than or equal
to x
+ */
+ private int max2Power(double x) {
+ double ans = 1;
+ int exponent = 0;
+ if (x > 1) {
+ while (ans * 2 <= x) {
+ ans = ans * 2;
+ exponent++;
+ }
+ } else {
+ while (ans > x) {
+ ans = ans / 2;
+ exponent--;
+ }
+ }
+ return exponent;
+ }
+
+ private ArrayList<Point> selectPoints(double a[]) {
+ // Keep the components with priority queue in the descending order of
energy
+ double sum2 = 0;
+ Point point;
+ PriorityQueue<Point> queue = new PriorityQueue<>(writeIndex);
+ for (int i = 0; i < writeIndex; i++) {
+ point = new Point(i, a[i]);
+ queue.add(point);
+ sum2 += point.getPower();
+ }
+ // Add components to keepList
+ this.beta = initBeta(sum2);
+ double systemError = sum2;
+ ArrayList<Point> keepList = new ArrayList<>();
+ int m = 0; // Number of reserved components
+ double roundingError = 0;
+ double reduceBits = Double.MAX_VALUE;
+ boolean first = true;
+ do {
+ while (systemError + roundingError > threshold * sum2) {
+ point = queue.poll();
+ if (point == null) {
+ systemError = 0;
+ break;
+ }
+ keepList.add(point);
+ systemError = systemError - point.getPower();
+ roundingError = Math.pow(2, this.beta * 2) * keepList.size();
+ }
+ double increaseBits = estimateIncreaseBits(keepList, m);
+ if (reduceBits <= increaseBits || systemError + roundingError >
threshold * sum2) {
+ if (!first) {
+ keepList = new ArrayList(keepList.subList(0, m));
+ this.beta--;
+ }
+ break;
+ }
+ // Increase quantification level
+ first = false;
+ m = keepList.size();
+ reduceBits = m;
+ this.beta++;
+ roundingError = Math.pow(2, this.beta * 2) * m;
+ } while (true);
+ return keepList;
+ }
+
+ /**
+ * Estimate the number of increased bits by reserving more components
+ *
+ * @param list The list of reserved components in this turn
+ * @param m The number of resereved components in last turn
+ * @return Estimated number of bits
+ */
+ private double estimateIncreaseBits(ArrayList<Point> list, int m) {
+ double bits = 0;
+ double eps = Math.pow(2, beta);
+ for (int i = m; i < list.size(); i++) {
+ bits += getValueWidth(writeIndex - 1); // Index
+ bits += getValueWidth(Math.round(Math.abs(list.get(i).getValue()) /
eps)); // Value
+ bits += 1; // Symbol
+ }
+ return bits;
+ }
+
+ protected class Point implements Comparable<Point> {
+
+ private final int index;
+ private final double value;
+
+ public Point(int index, double value) {
+ this.index = index;
+ this.value = value;
+ }
+
+ @Override
+ public int compareTo(Point o) {
+ return Double.compare(o.getPower(), this.getPower());
+ }
+
+ /** @return the index */
+ public int getIndex() {
+ return index;
+ }
+
+ /** @return the value */
+ public double getValue() {
+ return value;
+ }
+
+ public double getPower() {
+ return value * value;
+ }
+ }
+}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java
index 6d9cef8..913e23a 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.iotdb.tsfile.encoding.encoder;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
@@ -70,6 +69,8 @@ public abstract class TSEncodingBuilder {
return new GorillaV2();
case DICTIONARY:
return new Dictionary();
+ case FREQ:
+ return new Freq();
case ZIGZAG:
return new Zigzag();
default:
@@ -127,6 +128,68 @@ public abstract class TSEncodingBuilder {
}
}
+ /** for INT32, INT64, FLOAT, DOUBLE. */
+ public static class Freq extends TSEncodingBuilder {
+
+ private double snr =
TSFileDescriptor.getInstance().getConfig().getFreqEncodingSNR();
+ private int blockSize =
TSFileDescriptor.getInstance().getConfig().getFreqEncodingBlockSize();
+
+ @Override
+ public Encoder getEncoder(TSDataType type) {
+ switch (type) {
+ case INT32:
+ case INT64:
+ case FLOAT:
+ case DOUBLE:
+ return new FreqEncoder(blockSize, snr);
+ default:
+ throw new UnSupportedDataTypeException("FREQ doesn't support data
type: " + type);
+ }
+ }
+
+ @Override
+ public void initFromProps(Map<String, String> props) {
+ // set SNR from initialized map or default value if not set
+ if (props == null || !props.containsKey(FreqEncoder.FREQ_ENCODING_SNR)) {
+ snr = TSFileDescriptor.getInstance().getConfig().getFreqEncodingSNR();
+ } else {
+ try {
+ snr = Double.parseDouble(props.get(FreqEncoder.FREQ_ENCODING_SNR));
+ } catch (NumberFormatException e) {
+ logger.warn(
+ "The format of FREQ encoding SNR {} is not correct."
+ + " Using default FREQ encoding SNR.",
+ props.get(FreqEncoder.FREQ_ENCODING_SNR));
+ }
+ if (snr < 0) {
+ snr =
TSFileDescriptor.getInstance().getConfig().getFreqEncodingSNR();
+ logger.warn(
+ "cannot set FREQ encoding SNR to negative value, replaced with
default value:{}",
+ snr);
+ }
+ }
+ // set block size from initialized map or default value if not set
+ if (props == null ||
!props.containsKey(FreqEncoder.FREQ_ENCODING_BLOCK_SIZE)) {
+ blockSize =
TSFileDescriptor.getInstance().getConfig().getFreqEncodingBlockSize();
+ } else {
+ try {
+ blockSize =
Integer.parseInt(props.get(FreqEncoder.FREQ_ENCODING_BLOCK_SIZE));
+ } catch (NumberFormatException e) {
+ logger.warn(
+ "The format of FREQ encoding block size {} is not correct."
+ + " Using default FREQ encoding block size.",
+ props.get(FreqEncoder.FREQ_ENCODING_BLOCK_SIZE));
+ }
+ if (blockSize < 0) {
+ blockSize =
TSFileDescriptor.getInstance().getConfig().getFreqEncodingBlockSize();
+ logger.warn(
+ "cannot set FREQ encoding block size to negative value, replaced
with default value:{}",
+ blockSize);
+ }
+ }
+ }
+ }
+
/** for ENUMS, INT32, BOOLEAN, INT64, FLOAT, DOUBLE. */
public static class Rle extends TSEncodingBuilder {
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/TSEncoding.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/TSEncoding.java
index f42c301..59f7573 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/TSEncoding.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/TSEncoding.java
@@ -28,7 +28,8 @@ public enum TSEncoding {
GORILLA_V1((byte) 6),
REGULAR((byte) 7),
GORILLA((byte) 8),
- ZIGZAG((byte) 9);
+ ZIGZAG((byte) 9),
+ FREQ((byte) 10);
private final byte type;
@@ -68,6 +69,8 @@ public enum TSEncoding {
return TSEncoding.GORILLA;
case 9:
return TSEncoding.ZIGZAG;
+ case 10:
+ return TSEncoding.FREQ;
default:
throw new IllegalArgumentException("Invalid input: " + encoding);
}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BitConstructor.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BitConstructor.java
new file mode 100644
index 0000000..444927f
--- /dev/null
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BitConstructor.java
@@ -0,0 +1,93 @@
+/*
+ * Copyright 2021 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.iotdb.tsfile.utils;
+
+import org.eclipse.collections.impl.list.mutable.primitive.ByteArrayList;
+
+public class BitConstructor {
+
+ private static final int BITS_IN_A_BYTE = 8;
+ private static final long ALL_MASK = -1;
+ private final ByteArrayList data;
+ private byte cache = 0;
+ private int cnt = 0;
+
+ public BitConstructor() {
+ this.data = new ByteArrayList();
+ }
+
+ public BitConstructor(int initialCapacity) {
+ this.data = new ByteArrayList(initialCapacity);
+ }
+
+ public void add(long x, int len) {
+ x = x & ~(ALL_MASK << len); // Make sure that all bits expect the lowest
len bits of x are 0
+ while (len > 0) {
+ // Number of bits inserted into cache
+ int m = len + cnt >= BITS_IN_A_BYTE ? BITS_IN_A_BYTE - cnt : len;
+ len -= m;
+ cnt += m;
+ byte y = (byte) (x >> len);
+ y = (byte) (y << (BITS_IN_A_BYTE - cnt));
+ cache = (byte) (cache | y);
+ x = x & ~(ALL_MASK << len);
+ if (cnt == BITS_IN_A_BYTE) {
+ pad();
+ }
+ }
+ }
+
+ public byte[] toByteArray() {
+ byte[] ret;
+ if (cnt > 0) {
+ data.add(cache);
+ ret = data.toArray();
+ data.removeAtIndex(data.size() - 1);
+ } else {
+ ret = data.toArray();
+ }
+ return ret;
+ }
+
+ public void clear() {
+ data.clear();
+ cache = 0x00;
+ cnt = 0;
+ }
+
+ /** Fill the rest part of cache with 0 */
+ public void pad() {
+ if (cnt > 0) {
+ data.add(cache);
+ cache = 0x00;
+ cnt = 0;
+ }
+ }
+
+ public void add(byte[] bytes) {
+ if (cnt == 0) {
+ data.addAll(bytes);
+ } else {
+ for (int i = 0; i < bytes.length; i++) {
+ add(bytes[i], 8);
+ }
+ }
+ }
+
+ public int sizeInBytes() {
+ return data.size() + (cnt > 0 ? 1 : 0);
+ }
+}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BitReader.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BitReader.java
new file mode 100644
index 0000000..94926eb
--- /dev/null
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BitReader.java
@@ -0,0 +1,70 @@
+/*
+ * Copyright 2021 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.iotdb.tsfile.utils;
+
+import java.nio.ByteBuffer;
+
+public class BitReader {
+
+ private static final int BITS_IN_A_BYTE = 8;
+ private static final byte MASKS[] = {(byte) 0xff, 0x7f, 0x3f, 0x1f, 0x0f,
0x07, 0x03, 0x01};
+ private final ByteBuffer buffer;
+ private int bitCnt = BITS_IN_A_BYTE;
+ private byte cache = 0;
+
+ public BitReader(ByteBuffer buffer) {
+ this.buffer = buffer;
+ }
+
+ public long next(int len) {
+ long ret = 0;
+ while (len > 0) {
+ if (bitCnt == BITS_IN_A_BYTE) {
+ next();
+ }
+ // Number of bits read from the current byte
+ int m = len + bitCnt >= BITS_IN_A_BYTE ? BITS_IN_A_BYTE - bitCnt : len;
+ len -= m;
+ ret = ret << m;
+ byte y = (byte) (cache & MASKS[bitCnt]); // Truncate the low bits with &
+ y = (byte) ((y & 0xff) >>> (BITS_IN_A_BYTE - bitCnt - m)); // Logical
shift right
+ ret = ret | (y & 0xff);
+ bitCnt += m;
+ }
+ return ret;
+ }
+
+ public byte[] nextBytes(int len) {
+ byte[] ret = new byte[len];
+ if (bitCnt == BITS_IN_A_BYTE) {
+ buffer.get(ret);
+ } else {
+ for (int i = 0; i < len; i++) {
+ ret[i] = (byte) next(8);
+ }
+ }
+ return ret;
+ }
+
+ public void skip() {
+ this.bitCnt = BITS_IN_A_BYTE;
+ }
+
+ private void next() {
+ this.cache = buffer.get();
+ this.bitCnt = 0;
+ }
+}
diff --git
a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/FreqDecoderTest.java
b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/FreqDecoderTest.java
new file mode 100644
index 0000000..1f72b43
--- /dev/null
+++
b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/FreqDecoderTest.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.tsfile.encoding.decoder;
+
+import org.apache.iotdb.tsfile.encoding.encoder.FreqEncoder;
+
+import org.jtransforms.fft.DoubleFFT_1D;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Random;
+
+import static org.junit.Assert.assertTrue;
+
+public class FreqDecoderTest {
+
+ private static final int ROW_NUM = 1024;
+ ByteArrayOutputStream out;
+ private FreqEncoder writer;
+ private FreqDecoder reader;
+ private final Random ran = new Random();
+ private ByteBuffer buffer;
+
+ @Before
+ public void test() {
+ writer = new FreqEncoder();
+ reader = new FreqDecoder();
+ }
+
+ @Test
+ public void testSin() throws IOException {
+ reader.reset();
+ double[] data = new double[ROW_NUM];
+ for (int i = 0; i < ROW_NUM; i++) {
+ data[i] = Math.cos(0.25 * Math.PI * i);
+ }
+ double[] recover = shouldReadAndWrite(data, ROW_NUM);
+ assertTrue(SNR(data, recover, ROW_NUM) > 40);
+ }
+
+ @Test
+ public void testBoundInt() throws IOException {
+ reader.reset();
+ double[] data = new double[ROW_NUM];
+ for (int i = 2; i < 21; i++) {
+ boundInt(i, data);
+ }
+ }
+
+ @Test
+ public void testLongTail() throws IOException {
+ reader.reset();
+ double[] a = new double[ROW_NUM * 2];
+ for (int i = 0; i < ROW_NUM; i++) {
+ double amp = Math.exp(-i);
+ double theta = ran.nextDouble() * 2 * Math.PI;
+ a[i * 2] = amp * Math.cos(theta);
+ a[i * 2 + 1] = amp * Math.sin(theta);
+ }
+ DoubleFFT_1D fft = new DoubleFFT_1D(ROW_NUM);
+ fft.complexInverse(a, false);
+ double data[] = new double[ROW_NUM];
+ for (int i = 0; i < ROW_NUM; i++) {
+ data[i] = a[i * 2];
+ }
+ double[] recover = shouldReadAndWrite(data, ROW_NUM);
+ assertTrue(SNR(data, recover, ROW_NUM) > 40);
+ }
+
+ @Test
+ public void testMaxMin() throws IOException {
+ reader.reset();
+ double[] data = new double[ROW_NUM];
+ for (int i = 0; i < ROW_NUM; i++) {
+ data[i] = (i & 1) == 0 ? Long.MAX_VALUE : Long.MIN_VALUE;
+ }
+ double[] recover = shouldReadAndWrite(data, ROW_NUM);
+ assertTrue(SNR(data, recover, ROW_NUM) > 40);
+ }
+
+ @Test
+ public void testConstantInt() throws IOException {
+ reader.reset();
+ double[] data = new double[ROW_NUM];
+ for (int i = 0; i < 10; i++) {
+ constantInt(i, data);
+ }
+ }
+
+ private void boundInt(int power, double[] data) throws IOException {
+ reader.reset();
+ double maxn = 1 << power;
+ for (int i = 0; i < ROW_NUM; i++) {
+ data[i] = ran.nextDouble() * maxn;
+ }
+ double[] recover = shouldReadAndWrite(data, ROW_NUM);
+ assertTrue(SNR(data, recover, ROW_NUM) > 40);
+ }
+
+ private void constantInt(int value, double[] data) throws IOException {
+ reader.reset();
+ for (int i = 0; i < ROW_NUM; i++) {
+ data[i] = value;
+ }
+ double[] recover = shouldReadAndWrite(data, ROW_NUM);
+ assertTrue(SNR(data, recover, ROW_NUM) > 40);
+ }
+
+ private void writeData(double[] data, int length) {
+ for (int i = 0; i < length; i++) {
+ writer.encode(data[i], out);
+ }
+ writer.flush(out);
+ }
+
+ private double[] shouldReadAndWrite(double[] data, int length) throws
IOException {
+ out = new ByteArrayOutputStream();
+ writeData(data, length);
+ byte[] page = out.toByteArray();
+ buffer = ByteBuffer.wrap(page);
+ int i = 0;
+ double recover[] = new double[length];
+ while (reader.hasNext(buffer)) {
+ recover[i] = reader.readDouble(buffer);
+ i++;
+ }
+ return recover;
+ }
+
+ public double SNR(double[] gd, double[] x, int length) {
+ double noise_power = 0, signal_power = 0;
+ for (int i = 0; i < length; i++) {
+ noise_power += (gd[i] - x[i]) * (gd[i] - x[i]);
+ signal_power += gd[i] * gd[i];
+ }
+ if (noise_power == 0) {
+ return Double.POSITIVE_INFINITY;
+ } else {
+ return 10 * Math.log10(signal_power / noise_power);
+ }
+ }
+}
