This is an automated email from the ASF dual-hosted git repository.
sunzesong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 172db70 [IOTDB-938] Re-implement Gorilla encoding algorithm (#1856)
172db70 is described below
commit 172db70b957c3326a48f1cfe292ce3f8004dc64a
Author: Steve Yurong Su <[email protected]>
AuthorDate: Fri Oct 30 17:42:25 2020 +0800
[IOTDB-938] Re-implement Gorilla encoding algorithm (#1856)
* change TSEncoding.GORILLA to TSEncoding.GORILLA_V1
* add the implementation framework for GORILLA_V2
* support encoding INT32 using GORILLA_V2 encoder
* add the encoders of GORILLA_V2
* add the decoders of GORILLA_V2
* fix tests
* fix tests
* add copyright info in LICENSE
* add copyright info in modified files
* fix bugs and add ITs
* add IT
* update Gorilla encoding user doc
* fix code smells
* change the return type of the method readNextClearBit to byte
* rename default Gorilla encoding to GORILLA
---
LICENSE | 27 +-
client-cpp/src/main/IOTDBSession.h | 10 +-
client-py/src/utils/IoTDBConstants.py | 3 +-
docs/SystemDesign/TsFile/Format.md | 3 +-
docs/UserGuide/Concept/Encoding.md | 10 +-
docs/zh/SystemDesign/TsFile/Format.md | 3 +-
docs/zh/UserGuide/Concept/Encoding.md | 14 +-
.../resources/conf/iotdb-engine.properties | 2 +-
.../main/java/org/apache/iotdb/db/qp/Planner.java | 6 +-
.../iotdb/db/qp/strategy/LogicalGenerator.java | 2 +-
.../org/apache/iotdb/db/utils/SchemaUtils.java | 22 +-
.../iotdb/db/integration/IoTDBMultiSeriesIT.java | 5 +
.../iotdb/db/integration/IoTDBSimpleQueryIT.java | 53 +-
.../iotdb/tsfile/common/conf/TSFileConfig.java | 29 +-
.../tsfile/encoding/common/EncodingConfig.java | 42 --
.../iotdb/tsfile/encoding/decoder/Decoder.java | 139 ++---
...nDecoder.java => DoublePrecisionDecoderV1.java} | 20 +-
.../encoding/decoder/DoublePrecisionDecoderV2.java | 51 ++
.../{GorillaDecoder.java => GorillaDecoderV1.java} | 15 +-
.../tsfile/encoding/decoder/GorillaDecoderV2.java | 120 +++++
.../tsfile/encoding/decoder/IntGorillaDecoder.java | 88 +++
.../encoding/decoder/LongGorillaDecoder.java | 87 +++
...nDecoder.java => SinglePrecisionDecoderV1.java} | 21 +-
.../encoding/decoder/SinglePrecisionDecoderV2.java | 51 ++
...nEncoder.java => DoublePrecisionEncoderV1.java} | 17 +-
.../encoding/encoder/DoublePrecisionEncoderV2.java | 53 ++
.../{GorillaEncoder.java => GorillaEncoderV1.java} | 18 +-
.../tsfile/encoding/encoder/GorillaEncoderV2.java | 106 ++++
.../tsfile/encoding/encoder/IntGorillaEncoder.java | 152 ++++++
.../encoding/encoder/LongGorillaEncoder.java | 152 ++++++
...nEncoder.java => SinglePrecisionEncoderV1.java} | 16 +-
.../encoding/encoder/SinglePrecisionEncoderV2.java | 53 ++
.../tsfile/encoding/encoder/TSEncodingBuilder.java | 61 ++-
.../tsfile/file/metadata/enums/TSEncoding.java | 14 +-
...aDecoderTest.java => GorillaDecoderV1Test.java} | 34 +-
.../encoding/decoder/GorillaDecoderV2Test.java | 592 +++++++++++++++++++++
.../iotdb/tsfile/read/reader/PageReaderTest.java | 24 +-
.../iotdb/tsfile/utils/TsFileGeneratorForTest.java | 2 +-
.../iotdb/tsfile/write/TsFileReadWriteTest.java | 55 +-
39 files changed, 1871 insertions(+), 301 deletions(-)
diff --git a/LICENSE b/LICENSE
index b88d620..d84f2bd 100644
--- a/LICENSE
+++ b/LICENSE
@@ -200,18 +200,37 @@
See the License for the specific language governing permissions and
limitations under the License.
-
-================================================================
-APACHE IOTDB SUBCOMPONENTS:
+--------------------------------------------------------------------------------
+ APACHE IOTDB SUBCOMPONENTS
+--------------------------------------------------------------------------------
The following class is copied from maven-wrapper
(https://github.com/takari/maven-wrapper),
which is under Apache License 2.0:
./.mvn/wrapper/MavenWrapperDownloader.java
-------------
+--------------------------------------------------------------------------------
The following class is modified from Apache commons-collections
./tsfile/src/main/java/org/apache/iotdb/tsfile/utils/Murmur128Hash.java
Relevant pr is: https://github.com/apache/commons-collections/pull/83/
+
+--------------------------------------------------------------------------------
+
+The following files include code modified from Michael Burman's gorilla-tsc
project.
+
+./tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/GorillaEncoderV2.java
+./tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/IntGorillaEncoder.java
+./tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/LongGorillaEncoder.java
+./tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/SinglePrecisionEncoderV2.java
+./tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DoublePrecisionEncoderV2.java
+./tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/GorillaDecoderV2.java
+./tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/IntGorillaDecoder.java
+./tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/LongGorillaDecoder.java
+./tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/SinglePrecisionDecoderV2.java
+./tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DoublePrecisionDecoderV2.java
+
+Copyright: 2016-2018 Michael Burman and/or other contributors
+Project page: https://github.com/burmanm/gorilla-tsc
+License: http://www.apache.org/licenses/LICENSE-2.0
diff --git a/client-cpp/src/main/IOTDBSession.h
b/client-cpp/src/main/IOTDBSession.h
index cf05e36..6defcd8 100644
--- a/client-cpp/src/main/IOTDBSession.h
+++ b/client-cpp/src/main/IOTDBSession.h
@@ -103,7 +103,15 @@ namespace TSDataType{
}
namespace TSEncoding {
enum TSEncoding {
- PLAIN, PLAIN_DICTIONARY, RLE, DIFF, TS_2DIFF, BITMAP, GORILLA, REGULAR
+ PLAIN = 0,
+ PLAIN_DICTIONARY = 1,
+ RLE = 2,
+ DIFF = 3,
+ TS_2DIFF = 4,
+ BITMAP = 5,
+ GORILLA_V1 = 6,
+ REGULAR = 7,
+ GORILLA = 8
};
}
namespace TSStatusCode {
diff --git a/client-py/src/utils/IoTDBConstants.py
b/client-py/src/utils/IoTDBConstants.py
index f8b4f15..f053af7 100644
--- a/client-py/src/utils/IoTDBConstants.py
+++ b/client-py/src/utils/IoTDBConstants.py
@@ -37,8 +37,9 @@ class TSEncoding(Enum):
DIFF = 3
TS_2DIFF = 4
BITMAP = 5
- GORILLA = 6
+ GORILLA_V1 = 6
REGULAR = 7
+ GORILLA = 8
@unique
diff --git a/docs/SystemDesign/TsFile/Format.md
b/docs/SystemDesign/TsFile/Format.md
index e6b5fb7..c9b5ecb 100644
--- a/docs/SystemDesign/TsFile/Format.md
+++ b/docs/SystemDesign/TsFile/Format.md
@@ -50,8 +50,9 @@
- 3: DIFF
- 4: TS_2DIFF
- 5: BITMAP
- - 6: GORILLA
+ - 6: GORILLA_V1
- 7: REGULAR
+ - 8: GORILLA
- **Compressing Type Hardcode**
- 0: UNCOMPRESSED
- 1: SNAPPY
diff --git a/docs/UserGuide/Concept/Encoding.md
b/docs/UserGuide/Concept/Encoding.md
index 4d90104..b17a31d 100644
--- a/docs/UserGuide/Concept/Encoding.md
+++ b/docs/UserGuide/Concept/Encoding.md
@@ -41,7 +41,11 @@ Run-length encoding can also be used to encode
floating-point numbers, but it is
* GORILLA
-GORILLA encoding is more suitable for floating-point sequence with similar
values and is not recommended for sequence data with large fluctuations.
+GORILLA encoding is lossless. It is more suitable for numerical sequence with
similar values and is not recommended for sequence data with large fluctuations.
+
+Currently, there are two versions of GORILLA encoding implementation, it is
recommended to use `GORILLA` instead of `GORILLA_V1` (deprecated).
+
+Usage restrictions: When using GORILLA to encode INT32 data, you need to
ensure that there is no data point with the value `Integer.MIN_VALUE` in the
sequence. When using GORILLA to encode INT64 data, you need to ensure that
there is no data point with the value `Long.MIN_VALUE` in the sequence.
* REGULAR
@@ -58,8 +62,8 @@ The four encodings described in the previous sections are
applicable to differen
|Data Type |Supported Encoding|
|:---:|:---:|
|BOOLEAN| PLAIN, RLE|
-|INT32 |PLAIN, RLE, TS_2DIFF, REGULAR|
-|INT64 |PLAIN, RLE, TS_2DIFF, REGULAR|
+|INT32 |PLAIN, RLE, TS_2DIFF, REGULAR, GORILLA|
+|INT64 |PLAIN, RLE, TS_2DIFF, REGULAR, GORILLA|
|FLOAT |PLAIN, RLE, TS_2DIFF, GORILLA|
|DOUBLE |PLAIN, RLE, TS_2DIFF, GORILLA|
|TEXT |PLAIN|
diff --git a/docs/zh/SystemDesign/TsFile/Format.md
b/docs/zh/SystemDesign/TsFile/Format.md
index 0f5eb5c..c573eb6 100644
--- a/docs/zh/SystemDesign/TsFile/Format.md
+++ b/docs/zh/SystemDesign/TsFile/Format.md
@@ -49,8 +49,9 @@
- 3: DIFF
- 4: TS_2DIFF
- 5: BITMAP
- - 6: GORILLA
+ - 6: GORILLA_V1
- 7: REGULAR
+ - 8: GORILLA
- **压缩类型**
- 0: UNCOMPRESSED
- 1: SNAPPY
diff --git a/docs/zh/UserGuide/Concept/Encoding.md
b/docs/zh/UserGuide/Concept/Encoding.md
index 1366e63..1e0de13 100644
--- a/docs/zh/UserGuide/Concept/Encoding.md
+++ b/docs/zh/UserGuide/Concept/Encoding.md
@@ -41,7 +41,11 @@ PLAIN编码，默认的编码方式，即不编码，支持多种数据类型，
* GORILLA编码（GORILLA）
-GORILLA编码，比较适合编码前后值比较接近的浮点数序列，不适合编码前后波动较大的数据。
+GORILLA编码是一种无损编码，它比较适合编码前后值比较接近的数值序列，不适合编码前后波动较大的数据。
+
+当前系统中存在两个版本的GORILLA编码实现，推荐使用`GORILLA`，不推荐使用`GORILLA_V1`（已过时）。
+
+使用限制：使用Gorilla编码INT32数据时，需要保证序列中不存在值为`Integer.MIN_VALUE`的数据点；使用Gorilla编码INT64数据时，需要保证序列中不存在值为`Long.MIN_VALUE`的数据点。
* 定频数据编码 (REGULAR)
@@ -53,15 +57,15 @@ GORILLA编码，比较适合编码前后值比较接近的浮点数序列，不
前文介绍的四种编码适用于不同的数据类型，若对应关系错误，则无法正确创建时间序列。数据类型与支持其编码的编码方式对应关系总结如表格2-3。
-<center> **表格2-3 数据类型与支持其编码的对应关系**
+<div style="text-align: center;"> **表格2-3 数据类型与支持其编码的对应关系**
|数据类型 |支持的编码|
|:---:|:---:|
|BOOLEAN| PLAIN, RLE|
-|INT32 |PLAIN, RLE, TS_2DIFF, REGULAR|
-|INT64 |PLAIN, RLE, TS_2DIFF, REGULAR|
+|INT32 |PLAIN, RLE, TS_2DIFF, REGULAR, GORILLA|
+|INT64 |PLAIN, RLE, TS_2DIFF, REGULAR, GORILLA|
|FLOAT |PLAIN, RLE, TS_2DIFF, GORILLA|
|DOUBLE |PLAIN, RLE, TS_2DIFF, GORILLA|
|TEXT |PLAIN|
-</center>
+</div>
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties
b/server/src/assembly/resources/conf/iotdb-engine.properties
index db1e03b..24d1808 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -510,7 +510,7 @@ float_precision=2
time_encoder=TS_2DIFF
# Encoder of value series. default value is PLAIN.
-# For int, long data type, also supports TS_2DIFF and RLE(run-length encoding).
+# For int, long data type, also supports TS_2DIFF and RLE(run-length
encoding), REGULAR and GORILLA.
# For float, double data type, also supports TS_2DIFF, RLE(run-length
encoding) and GORILLA.
# For text data type, only supports PLAIN.
value_encoder=PLAIN
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/Planner.java
b/server/src/main/java/org/apache/iotdb/db/qp/Planner.java
index 3f4f84b..a92ff64 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/Planner.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/Planner.java
@@ -115,8 +115,10 @@ public class Planner {
filterOp.setIsSingle(true);
filterOp.setPathSet(pathSet);
- BasicFunctionOperator left = new
BasicFunctionOperator(SQLConstant.GREATERTHANOREQUALTO, timePath,
Long.toString(startTime));
- BasicFunctionOperator right = new
BasicFunctionOperator(SQLConstant.LESSTHAN, timePath, Long.toString(endTime));
+ BasicFunctionOperator left = new
BasicFunctionOperator(SQLConstant.GREATERTHANOREQUALTO,
+ timePath, Long.toString(startTime));
+ BasicFunctionOperator right = new
BasicFunctionOperator(SQLConstant.LESSTHAN, timePath,
+ Long.toString(endTime));
filterOp.addChildOperator(left);
filterOp.addChildOperator(right);
diff --git
a/server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalGenerator.java
b/server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalGenerator.java
index 6feb46a..fae8a3b 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalGenerator.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalGenerator.java
@@ -201,7 +201,7 @@ public class LogicalGenerator extends SqlBaseBaseListener {
private UpdateOperator updateOp;
private QueryOperator queryOp;
private DeleteDataOperator deleteDataOp;
- private static final String DELETE_RANGE_ERROR_MSG =
+ private static final String DELETE_RANGE_ERROR_MSG =
"For delete statement, where clause can only contain atomic expressions
like : " +
"time > XXX, time <= XXX, or two atomic expressions connected by 'AND'";
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java
b/server/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java
index ce31f07..e4a4d8d 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java
@@ -49,27 +49,33 @@ public class SchemaUtils {
}
- private static Map<TSDataType, Set<TSEncoding>> schemaChecker = new
EnumMap<>(TSDataType.class);
+ private static final Map<TSDataType, Set<TSEncoding>> schemaChecker = new
EnumMap<>(
+ TSDataType.class);
static {
Set<TSEncoding> booleanSet = new HashSet<>();
booleanSet.add(TSEncoding.PLAIN);
booleanSet.add(TSEncoding.RLE);
schemaChecker.put(TSDataType.BOOLEAN, booleanSet);
- Set<TSEncoding> int32Set = new HashSet<>();
- int32Set.add(TSEncoding.PLAIN);
- int32Set.add(TSEncoding.RLE);
- int32Set.add(TSEncoding.TS_2DIFF);
- int32Set.add(TSEncoding.REGULAR);
- schemaChecker.put(TSDataType.INT32, int32Set);
- schemaChecker.put(TSDataType.INT64, int32Set);
+
+ Set<TSEncoding> intSet = new HashSet<>();
+ intSet.add(TSEncoding.PLAIN);
+ intSet.add(TSEncoding.RLE);
+ intSet.add(TSEncoding.TS_2DIFF);
+ intSet.add(TSEncoding.REGULAR);
+ intSet.add(TSEncoding.GORILLA);
+ schemaChecker.put(TSDataType.INT32, intSet);
+ schemaChecker.put(TSDataType.INT64, intSet);
+
Set<TSEncoding> floatSet = new HashSet<>();
floatSet.add(TSEncoding.PLAIN);
floatSet.add(TSEncoding.RLE);
floatSet.add(TSEncoding.TS_2DIFF);
+ floatSet.add(TSEncoding.GORILLA_V1);
floatSet.add(TSEncoding.GORILLA);
schemaChecker.put(TSDataType.FLOAT, floatSet);
schemaChecker.put(TSDataType.DOUBLE, floatSet);
+
Set<TSEncoding> textSet = new HashSet<>();
textSet.add(TSEncoding.PLAIN);
schemaChecker.put(TSDataType.TEXT, textSet);
diff --git
a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBMultiSeriesIT.java
b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBMultiSeriesIT.java
index 89940e1..2538cc4 100644
---
a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBMultiSeriesIT.java
+++
b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBMultiSeriesIT.java
@@ -448,6 +448,7 @@ public class IoTDBMultiSeriesIT {
statement.execute("CREATE TIMESERIES root.ln.wf01.wt01.age WITH
DATATYPE=INT32, ENCODING=RLE");
statement.execute("CREATE TIMESERIES root.ln.wf01.wt01.salary WITH
DATATYPE=INT64");
statement.execute("CREATE TIMESERIES root.ln.wf01.wt01.score WITH
DATATYPE=FLOAT");
+ statement.execute("CREATE TIMESERIES root.ln.wf01.wt01.grade WITH
DATATYPE=DOUBLE");
ResultSet nameRs = statement.executeQuery("SHOW TIMESERIES
root.ln.wf01.wt01.name");
nameRs.next();
@@ -464,6 +465,10 @@ public class IoTDBMultiSeriesIT {
ResultSet scoreRs = statement.executeQuery("SHOW TIMESERIES
root.ln.wf01.wt01.score");
scoreRs.next();
Assert.assertTrue(TSEncoding.GORILLA.name().equalsIgnoreCase(scoreRs.getString(5)));
+
+ ResultSet gradeRs = statement.executeQuery("SHOW TIMESERIES
root.ln.wf01.wt01.grade");
+ gradeRs.next();
+
Assert.assertTrue(TSEncoding.GORILLA.name().equalsIgnoreCase(gradeRs.getString(5)));
}
}
}
diff --git
a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSimpleQueryIT.java
b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSimpleQueryIT.java
index b75197d..be8bbf8 100644
---
a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSimpleQueryIT.java
+++
b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSimpleQueryIT.java
@@ -28,6 +28,7 @@ import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
@@ -43,12 +44,6 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import java.sql.*;
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.junit.Assert.*;
-
public class IoTDBSimpleQueryIT {
@Before
@@ -75,8 +70,8 @@ public class IoTDBSimpleQueryIT {
e.printStackTrace();
}
- MeasurementMNode mNode = (MeasurementMNode)
MManager.getInstance().getNodeByPath(new PartialPath(
- "root.sg1.d0.s1"));
+ MeasurementMNode mNode = (MeasurementMNode) MManager.getInstance()
+ .getNodeByPath(new PartialPath("root.sg1.d0.s1"));
assertNull(mNode.getSchema().getProps());
}
@@ -96,7 +91,7 @@ public class IoTDBSimpleQueryIT {
}
resultSet = statement.executeQuery(
- "select count(*) from root where time >= 1 and time <= 100
group by ([0, 100), 20ms, 20ms)");
+ "select count(*) from root where time >= 1 and time <= 100 group
by ([0, 100), 20ms, 20ms)");
// has an empty time column
Assert.assertEquals(1, resultSet.getMetaData().getColumnCount());
while (resultSet.next()) {
@@ -125,8 +120,8 @@ public class IoTDBSimpleQueryIT {
}
resultSet = statement.executeQuery(
- "select count(*) from root where time >= 1 and time <= 100 "
- + "group by ([0, 100), 20ms, 20ms) align by device");
+ "select count(*) from root where time >= 1 and time <= 100 "
+ + "group by ([0, 100), 20ms, 20ms) align by device");
// has time and device columns
Assert.assertEquals(2, resultSet.getMetaData().getColumnCount());
while (resultSet.next()) {
@@ -201,7 +196,6 @@ public class IoTDBSimpleQueryIT {
statement.execute("flush");
-
int count = 0;
try (ResultSet resultSet = statement.executeQuery("show timeseries")) {
while (resultSet.next()) {
@@ -238,7 +232,6 @@ public class IoTDBSimpleQueryIT {
statement.execute("flush");
-
int count = 0;
try (ResultSet resultSet = statement.executeQuery("show timeseries")) {
while (resultSet.next()) {
@@ -275,9 +268,8 @@ public class IoTDBSimpleQueryIT {
statement.execute("flush");
-
int count = 0;
- try (ResultSet resultSet = statement.executeQuery("show timeseries")){
+ try (ResultSet resultSet = statement.executeQuery("show timeseries")) {
while (resultSet.next()) {
count++;
}
@@ -312,7 +304,6 @@ public class IoTDBSimpleQueryIT {
statement.execute("flush");
-
int count = 0;
try (ResultSet resultSet = statement.executeQuery("show timeseries limit
8")) {
while (resultSet.next()) {
@@ -381,7 +372,6 @@ public class IoTDBSimpleQueryIT {
statement.execute("flush");
-
long count = 0;
try (ResultSet resultSet = statement
.executeQuery("select s0 from root.sg1.d0 where s0 > 18")) {
@@ -413,7 +403,6 @@ public class IoTDBSimpleQueryIT {
assertTrue(e.getMessage().contains("s1"));
}
-
try (ResultSet resultSet = statement.executeQuery("select s0, s1 from
root.sg1.d0")) {
while (resultSet.next()) {
assertEquals(1, resultSet.getInt("root.sg1.d0.s0"));
@@ -486,7 +475,6 @@ public class IoTDBSimpleQueryIT {
statement.execute("flush");
-
long count = 0;
try (ResultSet resultSet = statement
@@ -529,7 +517,6 @@ public class IoTDBSimpleQueryIT {
statement.execute("delete from root.sg1.d0.s0 where time <= 15");
-
long count = 0;
try (ResultSet resultSet = statement.executeQuery("select * from root"))
{
@@ -584,14 +571,6 @@ public class IoTDBSimpleQueryIT {
}
try {
- statement.execute("CREATE TIMESERIES root.sg1.d1.s2 with
datatype=INT32, encoding=GORILLA");
- } catch (Exception e) {
- Assert.assertEquals(
- "303: org.apache.iotdb.db.exception.metadata.MetadataException:
encoding INT32 does not support GORILLA",
- e.getMessage());
- }
-
- try {
statement
.execute("CREATE TIMESERIES root.sg1.d1.s3 with datatype=DOUBLE,
encoding=REGULAR");
} catch (Exception e) {
@@ -616,19 +595,23 @@ public class IoTDBSimpleQueryIT {
@Test
public void testUseSameStatement() throws SQLException {
- try (Connection connection =
DriverManager.getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/",
"root", "root");
- Statement statement = connection.createStatement()) {
+ try (Connection connection = DriverManager
+ .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root",
"root");
+ Statement statement = connection.createStatement()) {
statement.execute("SET STORAGE GROUP TO root.sg1");
- statement.execute("CREATE TIMESERIES root.sg1.d0.s0 WITH DATATYPE=INT64,
ENCODING=RLE, COMPRESSOR=SNAPPY");
- statement.execute("CREATE TIMESERIES root.sg1.d0.s1 WITH DATATYPE=INT64,
ENCODING=RLE, COMPRESSOR=SNAPPY");
- statement.execute("CREATE TIMESERIES root.sg1.d1.s0 WITH DATATYPE=INT64,
ENCODING=RLE, COMPRESSOR=SNAPPY");
- statement.execute("CREATE TIMESERIES root.sg1.d1.s1 WITH DATATYPE=INT64,
ENCODING=RLE, COMPRESSOR=SNAPPY");
+ statement.execute(
+ "CREATE TIMESERIES root.sg1.d0.s0 WITH DATATYPE=INT64, ENCODING=RLE,
COMPRESSOR=SNAPPY");
+ statement.execute(
+ "CREATE TIMESERIES root.sg1.d0.s1 WITH DATATYPE=INT64, ENCODING=RLE,
COMPRESSOR=SNAPPY");
+ statement.execute(
+ "CREATE TIMESERIES root.sg1.d1.s0 WITH DATATYPE=INT64, ENCODING=RLE,
COMPRESSOR=SNAPPY");
+ statement.execute(
+ "CREATE TIMESERIES root.sg1.d1.s1 WITH DATATYPE=INT64, ENCODING=RLE,
COMPRESSOR=SNAPPY");
statement.execute("insert into root.sg1.d0(timestamp,s0,s1)
values(1,1,1)");
statement.execute("insert into root.sg1.d1(timestamp,s0,s1)
values(1000,1000,1000)");
statement.execute("insert into root.sg1.d0(timestamp,s0,s1)
values(10,10,10)");
-
List<ResultSet> resultSetList = new ArrayList<>();
ResultSet r1 = statement.executeQuery("select * from root.sg1.d0 where
time <= 1");
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
index caccc35..24e4dde 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
@@ -30,23 +30,28 @@ import org.apache.iotdb.tsfile.fileSystem.FSType;
*/
public class TSFileConfig implements Serializable {
- // Memory configuration
+ /**
+ * encoding configuration
+ */
public static final int RLE_MIN_REPEATED_NUM = 8;
public static final int RLE_MAX_REPEATED_NUM = 0x7FFF;
public static final int RLE_MAX_BIT_PACKED_NUM = 63;
- // Data type configuration
- // Gorilla encoding configuration
- public static final int FLOAT_LENGTH = 32;
- public static final int FLAOT_LEADING_ZERO_LENGTH = 5;
public static final int FLOAT_VALUE_LENGTH = 6;
+ public static final int DOUBLE_VALUE_LENGTH = 7;
- // Encoder configuration
- public static final int DOUBLE_LENGTH = 64;
- public static final int DOUBLE_LEADING_ZERO_LENGTH = 6;
+ public static final int VALUE_BITS_LENGTH_32BIT = 32;
+ public static final int LEADING_ZERO_BITS_LENGTH_32BIT = 5;
+ public static final int MEANINGFUL_XOR_BITS_LENGTH_32BIT = 5;
- // RLE configuration
- public static final int DOUBLE_VALUE_LENGTH = 7;
+ public static final int VALUE_BITS_LENGTH_64BIT = 64;
+ public static final int LEADING_ZERO_BITS_LENGTH_64BIT = 6;
+ public static final int MEANINGFUL_XOR_BITS_LENGTH_64BIT = 6;
+
+ public static final int GORILLA_ENCODING_ENDING_INTEGER = Integer.MIN_VALUE;
+ public static final long GORILLA_ENCODING_ENDING_LONG = Long.MIN_VALUE;
+ public static final float GORILLA_ENCODING_ENDING_FLOAT = Float.NaN;
+ public static final double GORILLA_ENCODING_ENDING_DOUBLE = Double.NaN;
/**
* String encoder with UTF-8 encodes a character to at most 4 bytes.
@@ -104,8 +109,8 @@ public class TSFileConfig implements Serializable {
private String timeEncoding = "TS_2DIFF";
/**
* Encoder of value series. default value is PLAIN. For int, long data type,
- * TsFile also supports TS_2DIFF and RLE(run-length encoding). For float,
double
- * data type, TsFile also supports TS_2DIFF, RLE(run-length encoding) and
+ * TsFile also supports TS_2DIFF, REGULAR, GORILLA and RLE(run-length
encoding).
+ * For float, double data type, TsFile also supports TS_2DIFF,
RLE(run-length encoding) and
* GORILLA. For text data type, TsFile only supports PLAIN.
*/
private String valueEncoder = "PLAIN";
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/common/EncodingConfig.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/common/EncodingConfig.java
deleted file mode 100644
index 7d62dff..0000000
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/common/EncodingConfig.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.tsfile.encoding.common;
-
-/**
- * This class defines several constants using in encoding algorithm.
- */
-public class EncodingConfig {
-
- private EncodingConfig() {
- throw new IllegalAccessError("Utility class");
- }
-
- /**
- * if number n repeats more than(>=) RLE_MAX_REPEATED_NUM times, use rle
- * encoding, otherwise use bit-packing.
- */
- public static final int RLE_MAX_REPEATED_NUM = 8;
-
- // when to start a new bit-pacing group
- public static final int RLE_MAX_BIT_PACKED_NUM = 63;
-
- // bit width for Bitmap Encoding
- public static final int BITMAP_BITWIDTH = 1;
-}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/Decoder.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/Decoder.java
index 76892a4..6bffedd 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/Decoder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/Decoder.java
@@ -47,103 +47,104 @@ public abstract class Decoder {
return type;
}
- /**
- * get Decoder object by type.
- *
- * @param type TSEncoding type
- * @param dataType TSDataType
- * @return Decoder object
- */
- public static Decoder getDecoderByType(TSEncoding type, TSDataType dataType)
{
- // PLA and DFT encoding are not supported in current version
- if (type == TSEncoding.PLAIN) {
- return new PlainDecoder(EndianType.BIG_ENDIAN);
- } else if (type == TSEncoding.RLE) {
- switch (dataType) {
- case BOOLEAN:
- case INT32:
- return new IntRleDecoder(EndianType.BIG_ENDIAN);
- case INT64:
- return new LongRleDecoder(EndianType.BIG_ENDIAN);
- case FLOAT:
- case DOUBLE:
- return new FloatDecoder(TSEncoding.valueOf(type.toString()),
dataType);
- default:
- throw new TsFileDecodingException(String.format(ERROR_MSG, type,
dataType));
- }
- } else if (type == TSEncoding.TS_2DIFF) {
- switch (dataType) {
- case INT32:
- return new DeltaBinaryDecoder.IntDeltaDecoder();
- case INT64:
- return new DeltaBinaryDecoder.LongDeltaDecoder();
- case FLOAT:
- case DOUBLE:
- return new FloatDecoder(TSEncoding.valueOf(type.toString()),
dataType);
- default:
- throw new TsFileDecodingException(
- String.format(ERROR_MSG, type, dataType));
- }
- } else if (type == TSEncoding.GORILLA) {
- switch (dataType) {
- case FLOAT:
- return new SinglePrecisionDecoder();
- case DOUBLE:
- return new DoublePrecisionDecoder();
- default:
- throw new TsFileDecodingException(
- String.format(ERROR_MSG, type, dataType));
- }
- } else if (type == TSEncoding.REGULAR) {
- switch (dataType) {
- case INT32:
- return new RegularDataDecoder.IntRegularDecoder();
- case INT64:
- return new RegularDataDecoder.LongRegularDecoder();
- default:
- throw new TsFileDecodingException(
- String.format(ERROR_MSG, type, dataType));
- }
- } else {
- throw new TsFileDecodingException(
- String.format(ERROR_MSG, type, dataType));
+ public static Decoder getDecoderByType(TSEncoding encoding, TSDataType
dataType) {
+ switch (encoding) {
+ case PLAIN:
+ return new PlainDecoder(EndianType.BIG_ENDIAN);
+ case RLE:
+ switch (dataType) {
+ case BOOLEAN:
+ case INT32:
+ return new IntRleDecoder(EndianType.BIG_ENDIAN);
+ case INT64:
+ return new LongRleDecoder(EndianType.BIG_ENDIAN);
+ case FLOAT:
+ case DOUBLE:
+ return new FloatDecoder(TSEncoding.valueOf(encoding.toString()),
dataType);
+ default:
+ throw new TsFileDecodingException(String.format(ERROR_MSG,
encoding, dataType));
+ }
+ case TS_2DIFF:
+ switch (dataType) {
+ case INT32:
+ return new DeltaBinaryDecoder.IntDeltaDecoder();
+ case INT64:
+ return new DeltaBinaryDecoder.LongDeltaDecoder();
+ case FLOAT:
+ case DOUBLE:
+ return new FloatDecoder(TSEncoding.valueOf(encoding.toString()),
dataType);
+ default:
+ throw new TsFileDecodingException(String.format(ERROR_MSG,
encoding, dataType));
+ }
+ case GORILLA_V1:
+ switch (dataType) {
+ case FLOAT:
+ return new SinglePrecisionDecoderV1();
+ case DOUBLE:
+ return new DoublePrecisionDecoderV1();
+ default:
+ throw new TsFileDecodingException(String.format(ERROR_MSG,
encoding, dataType));
+ }
+ case REGULAR:
+ switch (dataType) {
+ case INT32:
+ return new RegularDataDecoder.IntRegularDecoder();
+ case INT64:
+ return new RegularDataDecoder.LongRegularDecoder();
+ default:
+ throw new TsFileDecodingException(String.format(ERROR_MSG,
encoding, dataType));
+ }
+ case GORILLA:
+ switch (dataType) {
+ case FLOAT:
+ return new SinglePrecisionDecoderV2();
+ case DOUBLE:
+ return new DoublePrecisionDecoderV2();
+ case INT32:
+ return new IntGorillaDecoder();
+ case INT64:
+ return new LongGorillaDecoder();
+ default:
+ throw new TsFileDecodingException(String.format(ERROR_MSG,
encoding, dataType));
+ }
+ default:
+ throw new TsFileDecodingException(String.format(ERROR_MSG, encoding,
dataType));
}
}
public int readInt(ByteBuffer buffer) {
- throw new TsFileDecodingException("Method readInt is not supproted by
Decoder");
+ throw new TsFileDecodingException("Method readInt is not supported by
Decoder");
}
public boolean readBoolean(ByteBuffer buffer) {
- throw new TsFileDecodingException("Method readBoolean is not supproted by
Decoder");
+ throw new TsFileDecodingException("Method readBoolean is not supported by
Decoder");
}
public short readShort(ByteBuffer buffer) {
- throw new TsFileDecodingException("Method readShort is not supproted by
Decoder");
+ throw new TsFileDecodingException("Method readShort is not supported by
Decoder");
}
public long readLong(ByteBuffer buffer) {
- throw new TsFileDecodingException("Method readLong is not supproted by
Decoder");
+ throw new TsFileDecodingException("Method readLong is not supported by
Decoder");
}
public float readFloat(ByteBuffer buffer) {
- throw new TsFileDecodingException("Method readFloat is not supproted by
Decoder");
+ throw new TsFileDecodingException("Method readFloat is not supported by
Decoder");
}
public double readDouble(ByteBuffer buffer) {
- throw new TsFileDecodingException("Method readDouble is not supproted by
Decoder");
+ throw new TsFileDecodingException("Method readDouble is not supported by
Decoder");
}
public Binary readBinary(ByteBuffer buffer) {
- throw new TsFileDecodingException("Method readBinary is not supproted by
Decoder");
+ throw new TsFileDecodingException("Method readBinary is not supported by
Decoder");
}
public BigDecimal readBigDecimal(ByteBuffer buffer) {
- throw new TsFileDecodingException("Method readBigDecimal is not supproted
by Decoder");
+ throw new TsFileDecodingException("Method readBigDecimal is not supported
by Decoder");
}
public abstract boolean hasNext(ByteBuffer buffer) throws IOException;
public abstract void reset();
-
}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DoublePrecisionDecoder.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DoublePrecisionDecoderV1.java
similarity index 81%
rename from
tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DoublePrecisionDecoder.java
rename to
tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DoublePrecisionDecoderV1.java
index e25376a..221662c 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DoublePrecisionDecoder.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DoublePrecisionDecoderV1.java
@@ -31,12 +31,12 @@ import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
/**
* Decoder for value value using gorilla.
*/
-public class DoublePrecisionDecoder extends GorillaDecoder {
+public class DoublePrecisionDecoderV1 extends GorillaDecoderV1 {
- private static final Logger logger =
LoggerFactory.getLogger(DoublePrecisionDecoder.class);
+ private static final Logger logger =
LoggerFactory.getLogger(DoublePrecisionDecoderV1.class);
private long preValue;
- public DoublePrecisionDecoder() {
+ public DoublePrecisionDecoderV1() {
// do nothing
}
@@ -61,7 +61,7 @@ public class DoublePrecisionDecoder extends GorillaDecoder {
getNextValue(buffer);
return tmp;
} catch (IOException e) {
- logger.error("DoublePrecisionDecoder cannot read first double number",
e);
+ logger.error("DoublePrecisionDecoderV1 cannot read first double
number", e);
}
} else {
try {
@@ -69,7 +69,7 @@ public class DoublePrecisionDecoder extends GorillaDecoder {
getNextValue(buffer);
return tmp;
} catch (IOException e) {
- logger.error("DoublePrecisionDecoder cannot read following double
number", e);
+ logger.error("DoublePrecisionDecoderV1 cannot read following double
number", e);
}
}
return Double.NaN;
@@ -92,18 +92,20 @@ public class DoublePrecisionDecoder extends GorillaDecoder {
if (!nextFlag2) {
// case: '10'
long tmp = 0;
- for (int i = 0; i < TSFileConfig.DOUBLE_LENGTH - leadingZeroNum -
tailingZeroNum; i++) {
+ for (int i = 0; i < TSFileConfig.VALUE_BITS_LENGTH_64BIT -
leadingZeroNum - tailingZeroNum;
+ i++) {
long bit = readBit(buffer) ? 1 : 0;
- tmp |= (bit << (TSFileConfig.DOUBLE_LENGTH - 1 - leadingZeroNum - i));
+ tmp |= (bit << (TSFileConfig.VALUE_BITS_LENGTH_64BIT - 1 -
leadingZeroNum - i));
}
tmp ^= preValue;
preValue = tmp;
} else {
// case: '11'
- int leadingZeroNumTmp = readIntFromStream(buffer,
TSFileConfig.DOUBLE_LEADING_ZERO_LENGTH);
+ int leadingZeroNumTmp = readIntFromStream(buffer,
+ TSFileConfig.LEADING_ZERO_BITS_LENGTH_64BIT);
int lenTmp = readIntFromStream(buffer, TSFileConfig.DOUBLE_VALUE_LENGTH);
long tmp = readLongFromStream(buffer, lenTmp);
- tmp <<= (TSFileConfig.DOUBLE_LENGTH - leadingZeroNumTmp - lenTmp);
+ tmp <<= (TSFileConfig.VALUE_BITS_LENGTH_64BIT - leadingZeroNumTmp -
lenTmp);
tmp ^= preValue;
preValue = tmp;
}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DoublePrecisionDecoderV2.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DoublePrecisionDecoderV2.java
new file mode 100644
index 0000000..d911a8f
--- /dev/null
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DoublePrecisionDecoderV2.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.decoder;
+
+import static
org.apache.iotdb.tsfile.common.conf.TSFileConfig.GORILLA_ENCODING_ENDING_DOUBLE;
+
+import java.nio.ByteBuffer;
+
+/**
+ * <p> This class includes code modified from Michael Burman's gorilla-tsc
project.
+ *
+ * <p> Copyright: 2016-2018 Michael Burman and/or other contributors
+ * <p> Project page: https://github.com/burmanm/gorilla-tsc
+ * <p> License: http://www.apache.org/licenses/LICENSE-2.0
+ */
+public class DoublePrecisionDecoderV2 extends LongGorillaDecoder {
+
+ private static final long GORILLA_ENCODING_ENDING = Double
+ .doubleToRawLongBits(GORILLA_ENCODING_ENDING_DOUBLE);
+
+ @Override
+ public final double readDouble(ByteBuffer in) {
+ return Double.longBitsToDouble(readLong(in));
+ }
+
+ @Override
+ protected long cacheNext(ByteBuffer in) {
+ readNext(in);
+ if (storedValue == GORILLA_ENCODING_ENDING) {
+ hasNext = false;
+ }
+ return storedValue;
+ }
+}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/GorillaDecoder.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/GorillaDecoderV1.java
similarity index 92%
rename from
tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/GorillaDecoder.java
rename to
tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/GorillaDecoderV1.java
index 157122c..9f0694c 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/GorillaDecoder.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/GorillaDecoderV1.java
@@ -27,10 +27,10 @@ import org.slf4j.LoggerFactory;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
-public abstract class GorillaDecoder extends Decoder {
+public abstract class GorillaDecoderV1 extends Decoder {
protected static final int EOF = -1;
- private static final Logger logger =
LoggerFactory.getLogger(GorillaDecoder.class);
+ private static final Logger logger =
LoggerFactory.getLogger(GorillaDecoderV1.class);
// flag to indicate whether the first value is read from stream
protected boolean flag;
protected int leadingZeroNum;
@@ -44,8 +44,8 @@ public abstract class GorillaDecoder extends Decoder {
protected boolean nextFlag1;
protected boolean nextFlag2;
- public GorillaDecoder() {
- super(TSEncoding.GORILLA);
+ protected GorillaDecoderV1() {
+ super(TSEncoding.GORILLA_V1);
reset();
}
@@ -58,10 +58,7 @@ public abstract class GorillaDecoder extends Decoder {
@Override
public boolean hasNext(ByteBuffer buffer) throws IOException {
- if (buffer.remaining() > 0 || !isEnd) {
- return true;
- }
- return false;
+ return buffer.remaining() > 0 || !isEnd;
}
protected boolean isEmpty() {
@@ -124,7 +121,7 @@ public abstract class GorillaDecoder extends Decoder {
protected long readLongFromStream(ByteBuffer buffer, int len) throws
IOException {
long num = 0;
for (int i = 0; i < len; i++) {
- long bit = (long) (readBit(buffer) ? 1 : 0);
+ long bit = readBit(buffer) ? 1 : 0;
num |= bit << (len - 1 - i);
}
return num;
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/GorillaDecoderV2.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/GorillaDecoderV2.java
new file mode 100644
index 0000000..fc2d327
--- /dev/null
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/GorillaDecoderV2.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.decoder;
+
+import java.nio.ByteBuffer;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+
+/**
+ * <p> This class includes code modified from Michael Burman's gorilla-tsc
project.
+ *
+ * <p> Copyright: 2016-2018 Michael Burman and/or other contributors
+ * <p> Project page: https://github.com/burmanm/gorilla-tsc
+ * <p> License: http://www.apache.org/licenses/LICENSE-2.0
+ */
+public abstract class GorillaDecoderV2 extends Decoder {
+
+ protected boolean firstValueWasRead = false;
+ protected int storedLeadingZeros = Integer.MAX_VALUE;
+ protected int storedTrailingZeros = 0;
+ protected boolean hasNext = true;
+
+ private byte buffer = 0;
+ private int bitsLeft = 0;
+
+ protected GorillaDecoderV2() {
+ super(TSEncoding.GORILLA);
+ }
+
+ @Override
+ public final boolean hasNext(ByteBuffer in) {
+ return hasNext;
+ }
+
+ @Override
+ public void reset() {
+ firstValueWasRead = false;
+ storedLeadingZeros = Integer.MAX_VALUE;
+ storedTrailingZeros = 0;
+ hasNext = true;
+
+ buffer = 0;
+ bitsLeft = 0;
+ }
+
+ /**
+ * Reads the next bit and returns a boolean representing it.
+ *
+ * @return true if the next bit is 1, otherwise 0.
+ */
+ protected boolean readBit(ByteBuffer in) {
+ boolean bit = ((buffer >> (bitsLeft - 1)) & 1) == 1;
+ bitsLeft--;
+ flipByte(in);
+ return bit;
+ }
+
+ /**
+ * Reads a long from the next X bits that represent the least significant
bits in the long value.
+ *
+ * @param bits How many next bits are read from the stream
+ * @return long value that was read from the stream
+ */
+ protected long readLong(int bits, ByteBuffer in) {
+ long value = 0;
+ while (bits > 0) {
+ if (bits > bitsLeft || bits == Byte.SIZE) {
+ // Take only the bitsLeft "least significant" bits
+ byte d = (byte) (buffer & ((1 << bitsLeft) - 1));
+ value = (value << bitsLeft) + (d & 0xFF);
+ bits -= bitsLeft;
+ bitsLeft = 0;
+ } else {
+ // Shift to correct position and take only least significant bits
+ byte d = (byte) ((buffer >>> (bitsLeft - bits)) & ((1 << bits) - 1));
+ value = (value << bits) + (d & 0xFF);
+ bitsLeft -= bits;
+ bits = 0;
+ }
+ flipByte(in);
+ }
+ return value;
+ }
+
+ protected byte readNextClearBit(int maxBits, ByteBuffer in) {
+ byte value = 0x00;
+ for (int i = 0; i < maxBits; i++) {
+ value <<= 1;
+ if (readBit(in)) {
+ value |= 0x01;
+ } else {
+ break;
+ }
+ }
+ return value;
+ }
+
+ protected void flipByte(ByteBuffer in) {
+ if (bitsLeft == 0) {
+ buffer = in.get();
+ bitsLeft = Byte.SIZE;
+ }
+ }
+}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/IntGorillaDecoder.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/IntGorillaDecoder.java
new file mode 100644
index 0000000..bbb4f9f
--- /dev/null
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/IntGorillaDecoder.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.decoder;
+
+import static
org.apache.iotdb.tsfile.common.conf.TSFileConfig.GORILLA_ENCODING_ENDING_INTEGER;
+import static
org.apache.iotdb.tsfile.common.conf.TSFileConfig.LEADING_ZERO_BITS_LENGTH_32BIT;
+import static
org.apache.iotdb.tsfile.common.conf.TSFileConfig.MEANINGFUL_XOR_BITS_LENGTH_32BIT;
+import static
org.apache.iotdb.tsfile.common.conf.TSFileConfig.VALUE_BITS_LENGTH_32BIT;
+
+import java.nio.ByteBuffer;
+
+/**
+ * <p> This class includes code modified from Michael Burman's gorilla-tsc
project.
+ *
+ * <p> Copyright: 2016-2018 Michael Burman and/or other contributors
+ * <p> Project page: https://github.com/burmanm/gorilla-tsc
+ * <p> License: http://www.apache.org/licenses/LICENSE-2.0
+ */
+public class IntGorillaDecoder extends GorillaDecoderV2 {
+
+ protected int storedValue = 0;
+
+ @Override
+ public void reset() {
+ super.reset();
+ storedValue = 0;
+ }
+
+ @Override
+ public final int readInt(ByteBuffer in) {
+ int returnValue = storedValue;
+ if (!firstValueWasRead) {
+ flipByte(in);
+ storedValue = (int) readLong(VALUE_BITS_LENGTH_32BIT, in);
+ firstValueWasRead = true;
+ returnValue = storedValue;
+ }
+ cacheNext(in);
+ return returnValue;
+ }
+
+ protected int cacheNext(ByteBuffer in) {
+ readNext(in);
+ if (storedValue == GORILLA_ENCODING_ENDING_INTEGER) {
+ hasNext = false;
+ }
+ return storedValue;
+ }
+
+ @SuppressWarnings("squid:S128")
+ protected int readNext(ByteBuffer in) {
+ byte controlBits = readNextClearBit(2, in);
+
+ switch (controlBits) {
+ case 3: // case '11': use new leading and trailing zeros
+ storedLeadingZeros = (int) readLong(LEADING_ZERO_BITS_LENGTH_32BIT,
in);
+ byte significantBits = (byte)
readLong(MEANINGFUL_XOR_BITS_LENGTH_32BIT, in);
+ significantBits++;
+ storedTrailingZeros = VALUE_BITS_LENGTH_32BIT - significantBits -
storedLeadingZeros;
+ // missing break is intentional, we want to overflow to next one
+ case 2: // case '10': use stored leading and trailing zeros
+ int xor = (int) readLong(VALUE_BITS_LENGTH_32BIT - storedLeadingZeros
- storedTrailingZeros,
+ in);
+ xor <<= storedTrailingZeros;
+ storedValue ^= xor;
+ // missing break is intentional, we want to overflow to next one
+ default: // case '0': use stored value
+ return storedValue;
+ }
+ }
+}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/LongGorillaDecoder.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/LongGorillaDecoder.java
new file mode 100644
index 0000000..eb9380b
--- /dev/null
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/LongGorillaDecoder.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.decoder;
+
+import static
org.apache.iotdb.tsfile.common.conf.TSFileConfig.GORILLA_ENCODING_ENDING_LONG;
+import static
org.apache.iotdb.tsfile.common.conf.TSFileConfig.LEADING_ZERO_BITS_LENGTH_64BIT;
+import static
org.apache.iotdb.tsfile.common.conf.TSFileConfig.MEANINGFUL_XOR_BITS_LENGTH_64BIT;
+import static
org.apache.iotdb.tsfile.common.conf.TSFileConfig.VALUE_BITS_LENGTH_64BIT;
+
+import java.nio.ByteBuffer;
+
+/**
+ * <p> This class includes code modified from Michael Burman's gorilla-tsc
project.
+ *
+ * <p> Copyright: 2016-2018 Michael Burman and/or other contributors
+ * <p> Project page: https://github.com/burmanm/gorilla-tsc
+ * <p> License: http://www.apache.org/licenses/LICENSE-2.0
+ */
+public class LongGorillaDecoder extends GorillaDecoderV2 {
+
+ protected long storedValue = 0;
+
+ @Override
+ public void reset() {
+ super.reset();
+ storedValue = 0;
+ }
+
+ @Override
+ public final long readLong(ByteBuffer in) {
+ long returnValue = storedValue;
+ if (!firstValueWasRead) {
+ flipByte(in);
+ storedValue = readLong(VALUE_BITS_LENGTH_64BIT, in);
+ firstValueWasRead = true;
+ returnValue = storedValue;
+ }
+ cacheNext(in);
+ return returnValue;
+ }
+
+ protected long cacheNext(ByteBuffer in) {
+ readNext(in);
+ if (storedValue == GORILLA_ENCODING_ENDING_LONG) {
+ hasNext = false;
+ }
+ return storedValue;
+ }
+
+ @SuppressWarnings("squid:S128")
+ protected long readNext(ByteBuffer in) {
+ byte controlBits = readNextClearBit(2, in);
+
+ switch (controlBits) {
+ case 3: // case '11': use new leading and trailing zeros
+ storedLeadingZeros = (int) readLong(LEADING_ZERO_BITS_LENGTH_64BIT,
in);
+ byte significantBits = (byte)
readLong(MEANINGFUL_XOR_BITS_LENGTH_64BIT, in);
+ significantBits++;
+ storedTrailingZeros = VALUE_BITS_LENGTH_64BIT - significantBits -
storedLeadingZeros;
+ // missing break is intentional, we want to overflow to next one
+ case 2: // case '10': use stored leading and trailing zeros
+ long xor = readLong(VALUE_BITS_LENGTH_64BIT - storedLeadingZeros -
storedTrailingZeros, in);
+ xor <<= storedTrailingZeros;
+ storedValue ^= xor;
+ // missing break is intentional, we want to overflow to next one
+ default: // case '0': use stored value
+ return storedValue;
+ }
+ }
+}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/SinglePrecisionDecoder.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/SinglePrecisionDecoderV1.java
similarity index 81%
rename from
tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/SinglePrecisionDecoder.java
rename to
tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/SinglePrecisionDecoderV1.java
index 2841509..58f91f1 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/SinglePrecisionDecoder.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/SinglePrecisionDecoderV1.java
@@ -31,12 +31,12 @@ import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
/**
* Decoder for value value using gorilla.
*/
-public class SinglePrecisionDecoder extends GorillaDecoder {
+public class SinglePrecisionDecoderV1 extends GorillaDecoderV1 {
- private static final Logger logger =
LoggerFactory.getLogger(SinglePrecisionDecoder.class);
+ private static final Logger logger =
LoggerFactory.getLogger(SinglePrecisionDecoderV1.class);
private int preValue;
- public SinglePrecisionDecoder() {
+ public SinglePrecisionDecoderV1() {
// do nothing
}
@@ -57,7 +57,7 @@ public class SinglePrecisionDecoder extends GorillaDecoder {
getNextValue(buffer);
return tmp;
} catch (IOException e) {
- logger.error("SinglePrecisionDecoder cannot read first float number",
e);
+ logger.error("SinglePrecisionDecoderV1 cannot read first float
number", e);
}
} else {
try {
@@ -65,7 +65,7 @@ public class SinglePrecisionDecoder extends GorillaDecoder {
getNextValue(buffer);
return tmp;
} catch (IOException e) {
- logger.error("SinglePrecisionDecoder cannot read following float
number", e);
+ logger.error("SinglePrecisionDecoderV1 cannot read following float
number", e);
}
}
return Float.NaN;
@@ -88,18 +88,20 @@ public class SinglePrecisionDecoder extends GorillaDecoder {
if (!nextFlag2) {
// case: '10'
int tmp = 0;
- for (int i = 0; i < TSFileConfig.FLOAT_LENGTH - leadingZeroNum -
tailingZeroNum; i++) {
+ for (int i = 0; i < TSFileConfig.VALUE_BITS_LENGTH_32BIT -
leadingZeroNum - tailingZeroNum;
+ i++) {
int bit = readBit(buffer) ? 1 : 0;
- tmp |= bit << (TSFileConfig.FLOAT_LENGTH - 1 - leadingZeroNum - i);
+ tmp |= bit << (TSFileConfig.VALUE_BITS_LENGTH_32BIT - 1 -
leadingZeroNum - i);
}
tmp ^= preValue;
preValue = tmp;
} else {
// case: '11'
- int leadingZeroNumTmp = readIntFromStream(buffer,
TSFileConfig.FLAOT_LEADING_ZERO_LENGTH);
+ int leadingZeroNumTmp = readIntFromStream(buffer,
+ TSFileConfig.LEADING_ZERO_BITS_LENGTH_32BIT);
int lenTmp = readIntFromStream(buffer, TSFileConfig.FLOAT_VALUE_LENGTH);
int tmp = readIntFromStream(buffer, lenTmp);
- tmp <<= (TSFileConfig.FLOAT_LENGTH - leadingZeroNumTmp - lenTmp);
+ tmp <<= (TSFileConfig.VALUE_BITS_LENGTH_32BIT - leadingZeroNumTmp -
lenTmp);
tmp ^= preValue;
preValue = tmp;
}
@@ -109,5 +111,4 @@ public class SinglePrecisionDecoder extends GorillaDecoder {
isEnd = true;
}
}
-
}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/SinglePrecisionDecoderV2.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/SinglePrecisionDecoderV2.java
new file mode 100644
index 0000000..a44c890
--- /dev/null
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/SinglePrecisionDecoderV2.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.decoder;
+
+import static
org.apache.iotdb.tsfile.common.conf.TSFileConfig.GORILLA_ENCODING_ENDING_FLOAT;
+
+import java.nio.ByteBuffer;
+
+/**
+ * <p> This class includes code modified from Michael Burman's gorilla-tsc
project.
+ *
+ * <p> Copyright: 2016-2018 Michael Burman and/or other contributors
+ * <p> Project page: https://github.com/burmanm/gorilla-tsc
+ * <p> License: http://www.apache.org/licenses/LICENSE-2.0
+ */
+public class SinglePrecisionDecoderV2 extends IntGorillaDecoder {
+
+ private static final int GORILLA_ENCODING_ENDING = Float
+ .floatToRawIntBits(GORILLA_ENCODING_ENDING_FLOAT);
+
+ @Override
+ public final float readFloat(ByteBuffer in) {
+ return Float.intBitsToFloat(readInt(in));
+ }
+
+ @Override
+ protected int cacheNext(ByteBuffer in) {
+ readNext(in);
+ if (storedValue == GORILLA_ENCODING_ENDING) {
+ hasNext = false;
+ }
+ return storedValue;
+ }
+}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DoublePrecisionEncoder.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DoublePrecisionEncoderV1.java
similarity index 83%
rename from
tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DoublePrecisionEncoder.java
rename to
tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DoublePrecisionEncoderV1.java
index 1afc377..e038d24 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DoublePrecisionEncoder.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DoublePrecisionEncoderV1.java
@@ -26,11 +26,11 @@ import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
/**
* Encoder for int value using gorilla encoding.
*/
-public class DoublePrecisionEncoder extends GorillaEncoder {
+public class DoublePrecisionEncoderV1 extends GorillaEncoderV1 {
private long preValue;
- public DoublePrecisionEncoder() {
+ public DoublePrecisionEncoderV1() {
// do nothing
}
@@ -62,16 +62,19 @@ public class DoublePrecisionEncoder extends GorillaEncoder {
// and last tailingZeroNum '0'
writeBit(true, out);
writeBit(false, out);
- writeBits(tmp, out, TSFileConfig.DOUBLE_LENGTH - 1 - leadingZeroNum,
tailingZeroNum);
+ writeBits(tmp, out, TSFileConfig.VALUE_BITS_LENGTH_64BIT - 1 -
leadingZeroNum,
+ tailingZeroNum);
} else {
// case: write '11', leading zero num of value, effective bits len
and effective
// bit value
writeBit(true, out);
writeBit(true, out);
- writeBits(leadingZeroNumTmp, out,
TSFileConfig.DOUBLE_LEADING_ZERO_LENGTH - 1, 0);
- writeBits((long) TSFileConfig.DOUBLE_LENGTH - leadingZeroNumTmp -
tailingZeroNumTmp, out,
- TSFileConfig.DOUBLE_VALUE_LENGTH - 1, 0);
- writeBits(tmp, out, TSFileConfig.DOUBLE_LENGTH - 1 -
leadingZeroNumTmp, tailingZeroNumTmp);
+ writeBits(leadingZeroNumTmp, out,
TSFileConfig.LEADING_ZERO_BITS_LENGTH_64BIT - 1, 0);
+ writeBits(
+ (long) TSFileConfig.VALUE_BITS_LENGTH_64BIT - leadingZeroNumTmp
- tailingZeroNumTmp,
+ out, TSFileConfig.DOUBLE_VALUE_LENGTH - 1, 0);
+ writeBits(tmp, out, TSFileConfig.VALUE_BITS_LENGTH_64BIT - 1 -
leadingZeroNumTmp,
+ tailingZeroNumTmp);
}
}
preValue = nextValue;
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DoublePrecisionEncoderV2.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DoublePrecisionEncoderV2.java
new file mode 100644
index 0000000..92f3b04
--- /dev/null
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DoublePrecisionEncoderV2.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.encoder;
+
+import static
org.apache.iotdb.tsfile.common.conf.TSFileConfig.GORILLA_ENCODING_ENDING_DOUBLE;
+
+import java.io.ByteArrayOutputStream;
+
+/**
+ * <p> This class includes code modified from Michael Burman's gorilla-tsc
project.
+ *
+ * <p> Copyright: 2016-2018 Michael Burman and/or other contributors
+ * <p> Project page: https://github.com/burmanm/gorilla-tsc
+ * <p> License: http://www.apache.org/licenses/LICENSE-2.0
+ */
+public class DoublePrecisionEncoderV2 extends LongGorillaEncoder {
+
+ @Override
+ public final void encode(double value, ByteArrayOutputStream out) {
+ encode(Double.doubleToRawLongBits(value), out);
+ }
+
+ @Override
+ public void flush(ByteArrayOutputStream out) {
+ // ending stream
+ encode(GORILLA_ENCODING_ENDING_DOUBLE, out);
+
+ // flip the byte no matter it is empty or not
+ // the empty ending byte is necessary when decoding
+ bitsLeft = 0;
+ flipByte(out);
+
+ // the encoder may be reused, so let us reset it
+ reset();
+ }
+}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/GorillaEncoder.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/GorillaEncoderV1.java
similarity index 88%
rename from
tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/GorillaEncoder.java
rename to
tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/GorillaEncoderV1.java
index 3fb0938..2c8bd4a 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/GorillaEncoder.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/GorillaEncoderV1.java
@@ -26,7 +26,7 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
* Gorilla encoding. For more information about how it works, please see
* http://www.vldb.org/pvldb/vol8/p1816-teller.pdf
*/
-public abstract class GorillaEncoder extends Encoder {
+public abstract class GorillaEncoderV1 extends Encoder {
// flag to indicate whether the first value is saved
protected boolean flag;
@@ -37,8 +37,8 @@ public abstract class GorillaEncoder extends Encoder {
// number of bits remaining in buffer
protected int numberLeftInBuffer;
- public GorillaEncoder() {
- super(TSEncoding.GORILLA);
+ protected GorillaEncoderV1() {
+ super(TSEncoding.GORILLA_V1);
this.flag = false;
}
@@ -57,19 +57,11 @@ public abstract class GorillaEncoder extends Encoder {
}
protected void writeBit(int i, ByteArrayOutputStream out) {
- if (i == 0) {
- writeBit(false, out);
- } else {
- writeBit(true, out);
- }
+ writeBit(i != 0, out);
}
protected void writeBit(long i, ByteArrayOutputStream out) {
- if (i == 0) {
- writeBit(false, out);
- } else {
- writeBit(true, out);
- }
+ writeBit(i != 0, out);
}
protected void clearBuffer(ByteArrayOutputStream out) {
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/GorillaEncoderV2.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/GorillaEncoderV2.java
new file mode 100644
index 0000000..add9cfa
--- /dev/null
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/GorillaEncoderV2.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.encoder;
+
+import java.io.ByteArrayOutputStream;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+
+/**
+ * <p> This class includes code modified from Michael Burman's gorilla-tsc
project.
+ *
+ * <p> Copyright: 2016-2018 Michael Burman and/or other contributors
+ * <p> Project page: https://github.com/burmanm/gorilla-tsc
+ * <p> License: http://www.apache.org/licenses/LICENSE-2.0
+ */
+public abstract class GorillaEncoderV2 extends Encoder {
+
+ protected boolean firstValueWasWritten = false;
+ protected int storedLeadingZeros = Integer.MAX_VALUE;
+ protected int storedTrailingZeros = 0;
+
+ private byte buffer = 0;
+ protected int bitsLeft = Byte.SIZE;
+
+ protected GorillaEncoderV2() {
+ super(TSEncoding.GORILLA);
+ }
+
+ @Override
+ public final long getMaxByteSize() {
+ return 0;
+ }
+
+ protected void reset() {
+ firstValueWasWritten = false;
+ storedLeadingZeros = Integer.MAX_VALUE;
+ storedTrailingZeros = 0;
+
+ buffer = 0;
+ bitsLeft = Byte.SIZE;
+ }
+
+ /**
+ * Stores a 0 and increases the count of bits by 1
+ */
+ protected void skipBit(ByteArrayOutputStream out) {
+ bitsLeft--;
+ flipByte(out);
+ }
+
+ /**
+ * Stores a 1 and increases the count of bits by 1
+ */
+ protected void writeBit(ByteArrayOutputStream out) {
+ buffer |= (1 << (bitsLeft - 1));
+ bitsLeft--;
+ flipByte(out);
+ }
+
+ /**
+ * Writes the given long value using the defined amount of least significant
bits.
+ *
+ * @param value The long value to be written
+ * @param bits How many bits are stored to the stream
+ */
+ protected void writeBits(long value, int bits, ByteArrayOutputStream out) {
+ while (bits > 0) {
+ int shift = bits - bitsLeft;
+ if (shift >= 0) {
+ buffer |= (byte) ((value >> shift) & ((1 << bitsLeft) - 1));
+ bits -= bitsLeft;
+ bitsLeft = 0;
+ } else {
+ shift = bitsLeft - bits;
+ buffer |= (byte) (value << shift);
+ bitsLeft -= bits;
+ bits = 0;
+ }
+ flipByte(out);
+ }
+ }
+
+ protected void flipByte(ByteArrayOutputStream out) {
+ if (bitsLeft == 0) {
+ out.write(buffer);
+ buffer = 0;
+ bitsLeft = Byte.SIZE;
+ }
+ }
+}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/IntGorillaEncoder.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/IntGorillaEncoder.java
new file mode 100644
index 0000000..1b412ec
--- /dev/null
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/IntGorillaEncoder.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.encoder;
+
+import static
org.apache.iotdb.tsfile.common.conf.TSFileConfig.GORILLA_ENCODING_ENDING_INTEGER;
+import static
org.apache.iotdb.tsfile.common.conf.TSFileConfig.LEADING_ZERO_BITS_LENGTH_32BIT;
+import static
org.apache.iotdb.tsfile.common.conf.TSFileConfig.MEANINGFUL_XOR_BITS_LENGTH_32BIT;
+import static
org.apache.iotdb.tsfile.common.conf.TSFileConfig.VALUE_BITS_LENGTH_32BIT;
+
+import java.io.ByteArrayOutputStream;
+
+/**
+ * <p> This class includes code modified from Michael Burman's gorilla-tsc
project.
+ *
+ * <p> Copyright: 2016-2018 Michael Burman and/or other contributors
+ * <p> Project page: https://github.com/burmanm/gorilla-tsc
+ * <p> License: http://www.apache.org/licenses/LICENSE-2.0
+ */
+public class IntGorillaEncoder extends GorillaEncoderV2 {
+
+ private static final int ONE_ITEM_MAX_SIZE =
+ (2 + LEADING_ZERO_BITS_LENGTH_32BIT + MEANINGFUL_XOR_BITS_LENGTH_32BIT
+ + VALUE_BITS_LENGTH_32BIT) / Byte.SIZE + 1;
+
+ private int storedValue = 0;
+
+ @Override
+ public final int getOneItemMaxSize() {
+ return ONE_ITEM_MAX_SIZE;
+ }
+
+ @Override
+ public final void encode(int value, ByteArrayOutputStream out) {
+ if (firstValueWasWritten) {
+ compressValue(value, out);
+ } else {
+ writeFirst(value, out);
+ firstValueWasWritten = true;
+ }
+ }
+
+ @Override
+ public void flush(ByteArrayOutputStream out) {
+ // ending stream
+ encode(GORILLA_ENCODING_ENDING_INTEGER, out);
+
+ // flip the byte no matter it is empty or not
+ // the empty ending byte is necessary when decoding
+ bitsLeft = 0;
+ flipByte(out);
+
+ // the encoder may be reused, so let us reset it
+ reset();
+ }
+
+ @Override
+ protected void reset() {
+ super.reset();
+ storedValue = 0;
+ }
+
+ private void writeFirst(int value, ByteArrayOutputStream out) {
+ storedValue = value;
+ writeBits(value, VALUE_BITS_LENGTH_32BIT, out);
+ }
+
+ private void compressValue(int value, ByteArrayOutputStream out) {
+ int xor = storedValue ^ value;
+ storedValue = value;
+
+ if (xor == 0) {
+ skipBit(out);
+ } else {
+ writeBit(out);
+
+ int leadingZeros = Integer.numberOfLeadingZeros(xor);
+ int trailingZeros = Integer.numberOfTrailingZeros(xor);
+ if (leadingZeros >= storedLeadingZeros && trailingZeros >=
storedTrailingZeros) {
+ writeExistingLeading(xor, out);
+ } else {
+ writeNewLeading(xor, leadingZeros, trailingZeros, out);
+ }
+ }
+ }
+
+ /**
+ * If there at least as many leading zeros and as many trailing zeros as
previous value, control
+ * bit = 0 (type a)
+ * <p>
+ * store the meaningful XORed value
+ *
+ * @param xor XOR between previous value and current
+ */
+ private void writeExistingLeading(int xor, ByteArrayOutputStream out) {
+ skipBit(out);
+
+ int significantBits = VALUE_BITS_LENGTH_32BIT - storedLeadingZeros -
storedTrailingZeros;
+ writeBits(xor >>> storedTrailingZeros, significantBits, out);
+ }
+
+ /**
+ * Stores the length of the number of leading zeros in the next 5 bits
+ * <p>
+ * Stores the length of the meaningful XORed value in the next 5 bits
+ * <p>
+ * Stores the meaningful bits of the XORed value
+ * <p>
+ * (type b)
+ *
+ * @param xor XOR between previous value and current
+ * @param leadingZeros New leading zeros
+ * @param trailingZeros New trailing zeros
+ */
+ private void writeNewLeading(int xor, int leadingZeros, int trailingZeros,
+ ByteArrayOutputStream out) {
+ writeBit(out);
+
+ int significantBits = VALUE_BITS_LENGTH_32BIT - leadingZeros -
trailingZeros;
+
+ // Number of leading zeros in the next 5 bits
+ // Maximum number of leadingZeros is stored with 5 bits to allow up to 31
leading zeros, which
+ // are necessary when storing int values
+ // Note that in this method the number of leading zeros won't be 32
+ writeBits(leadingZeros, LEADING_ZERO_BITS_LENGTH_32BIT, out);
+ // Length of meaningful bits in the next 5 bits
+ // Note that in this method the number of meaningful bits is always
positive and could be 32,
+ // so we have to use (significantBits - 1) in storage
+ writeBits((long) significantBits - 1, MEANINGFUL_XOR_BITS_LENGTH_32BIT,
out);
+ // Store the meaningful bits of XOR
+ writeBits(xor >>> trailingZeros, significantBits, out);
+
+ storedLeadingZeros = leadingZeros;
+ storedTrailingZeros = trailingZeros;
+ }
+}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/LongGorillaEncoder.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/LongGorillaEncoder.java
new file mode 100644
index 0000000..6058a45
--- /dev/null
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/LongGorillaEncoder.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.encoder;
+
+import static
org.apache.iotdb.tsfile.common.conf.TSFileConfig.GORILLA_ENCODING_ENDING_LONG;
+import static
org.apache.iotdb.tsfile.common.conf.TSFileConfig.LEADING_ZERO_BITS_LENGTH_64BIT;
+import static
org.apache.iotdb.tsfile.common.conf.TSFileConfig.MEANINGFUL_XOR_BITS_LENGTH_64BIT;
+import static
org.apache.iotdb.tsfile.common.conf.TSFileConfig.VALUE_BITS_LENGTH_64BIT;
+
+import java.io.ByteArrayOutputStream;
+
+/**
+ * <p> This class includes code modified from Michael Burman's gorilla-tsc
project.
+ *
+ * <p> Copyright: 2016-2018 Michael Burman and/or other contributors
+ * <p> Project page: https://github.com/burmanm/gorilla-tsc
+ * <p> License: http://www.apache.org/licenses/LICENSE-2.0
+ */
+public class LongGorillaEncoder extends GorillaEncoderV2 {
+
+ private static final int ONE_ITEM_MAX_SIZE =
+ (2 + LEADING_ZERO_BITS_LENGTH_64BIT + MEANINGFUL_XOR_BITS_LENGTH_64BIT
+ + VALUE_BITS_LENGTH_64BIT) / Byte.SIZE + 1;
+
+ private long storedValue = 0;
+
+ @Override
+ public final int getOneItemMaxSize() {
+ return ONE_ITEM_MAX_SIZE;
+ }
+
+ @Override
+ public final void encode(long value, ByteArrayOutputStream out) {
+ if (firstValueWasWritten) {
+ compressValue(value, out);
+ } else {
+ writeFirst(value, out);
+ firstValueWasWritten = true;
+ }
+ }
+
+ @Override
+ public void flush(ByteArrayOutputStream out) {
+ // ending stream
+ encode(GORILLA_ENCODING_ENDING_LONG, out);
+
+ // flip the byte no matter it is empty or not
+ // the empty ending byte is necessary when decoding
+ bitsLeft = 0;
+ flipByte(out);
+
+ // the encoder may be reused, so let us reset it
+ reset();
+ }
+
+ @Override
+ protected void reset() {
+ super.reset();
+ storedValue = 0;
+ }
+
+ private void writeFirst(long value, ByteArrayOutputStream out) {
+ storedValue = value;
+ writeBits(value, VALUE_BITS_LENGTH_64BIT, out);
+ }
+
+ private void compressValue(long value, ByteArrayOutputStream out) {
+ long xor = storedValue ^ value;
+ storedValue = value;
+
+ if (xor == 0) {
+ skipBit(out);
+ } else {
+ writeBit(out);
+
+ int leadingZeros = Long.numberOfLeadingZeros(xor);
+ int trailingZeros = Long.numberOfTrailingZeros(xor);
+ if (leadingZeros >= storedLeadingZeros && trailingZeros >=
storedTrailingZeros) {
+ writeExistingLeading(xor, out);
+ } else {
+ writeNewLeading(xor, leadingZeros, trailingZeros, out);
+ }
+ }
+ }
+
+ /**
+ * If there at least as many leading zeros and as many trailing zeros as
previous value, control
+ * bit = 0 (type a)
+ * <p>
+ * store the meaningful XORed value
+ *
+ * @param xor XOR between previous value and current
+ */
+ private void writeExistingLeading(long xor, ByteArrayOutputStream out) {
+ skipBit(out);
+
+ int significantBits = VALUE_BITS_LENGTH_64BIT - storedLeadingZeros -
storedTrailingZeros;
+ writeBits(xor >>> storedTrailingZeros, significantBits, out);
+ }
+
+ /**
+ * Stores the length of the number of leading zeros in the next 6 bits
+ * <p>
+ * Stores the length of the meaningful XORed value in the next 6 bits
+ * <p>
+ * Stores the meaningful bits of the XORed value
+ * <p>
+ * (type b)
+ *
+ * @param xor XOR between previous value and current
+ * @param leadingZeros New leading zeros
+ * @param trailingZeros New trailing zeros
+ */
+ private void writeNewLeading(long xor, int leadingZeros, int trailingZeros,
+ ByteArrayOutputStream out) {
+ writeBit(out);
+
+ int significantBits = VALUE_BITS_LENGTH_64BIT - leadingZeros -
trailingZeros;
+
+ // Number of leading zeros in the next 6 bits
+ // Different from original, maximum number of leadingZeros is stored with
6 (original 5) bits to
+ // allow up to 63 leading zeros, which are necessary when storing long
values
+ // Note that in this method the number of leading zeros won't be 64
+ writeBits(leadingZeros, LEADING_ZERO_BITS_LENGTH_64BIT, out);
+ // Length of meaningful bits in the next 6 bits
+ // Note that in this method the number of meaningful bits is always
positive and could be 64,
+ // so we have to use (significantBits - 1) in storage
+ writeBits((long) significantBits - 1, MEANINGFUL_XOR_BITS_LENGTH_64BIT,
out);
+ // Store the meaningful bits of XOR
+ writeBits(xor >>> trailingZeros, significantBits, out);
+
+ storedLeadingZeros = leadingZeros;
+ storedTrailingZeros = trailingZeros;
+ }
+}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/SinglePrecisionEncoder.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/SinglePrecisionEncoderV1.java
similarity index 84%
rename from
tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/SinglePrecisionEncoder.java
rename to
tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/SinglePrecisionEncoderV1.java
index 057ada4..32ead4a 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/SinglePrecisionEncoder.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/SinglePrecisionEncoderV1.java
@@ -27,11 +27,11 @@ import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
/**
* Encoder for int value using gorilla encoding.
*/
-public class SinglePrecisionEncoder extends GorillaEncoder {
+public class SinglePrecisionEncoderV1 extends GorillaEncoderV1 {
private int preValue;
- public SinglePrecisionEncoder() {
+ public SinglePrecisionEncoderV1() {
// allowed do nothing
}
@@ -60,16 +60,18 @@ public class SinglePrecisionEncoder extends GorillaEncoder {
// last tailingZeroNum '0'
writeBit(true, out);
writeBit(false, out);
- writeBits(tmp, out, TSFileConfig.FLOAT_LENGTH - 1 - leadingZeroNum,
tailingZeroNum);
+ writeBits(tmp, out, TSFileConfig.VALUE_BITS_LENGTH_32BIT - 1 -
leadingZeroNum,
+ tailingZeroNum);
} else {
// case: write '11', leading zero num of value, effective bits len
and effective
// bit value
writeBit(true, out);
writeBit(true, out);
- writeBits(leadingZeroNumTmp, out,
TSFileConfig.FLAOT_LEADING_ZERO_LENGTH - 1, 0);
- writeBits(TSFileConfig.FLOAT_LENGTH - leadingZeroNumTmp -
tailingZeroNumTmp, out,
- TSFileConfig.FLOAT_VALUE_LENGTH - 1, 0);
- writeBits(tmp, out, TSFileConfig.FLOAT_LENGTH - 1 -
leadingZeroNumTmp, tailingZeroNumTmp);
+ writeBits(leadingZeroNumTmp, out,
TSFileConfig.LEADING_ZERO_BITS_LENGTH_32BIT - 1, 0);
+ writeBits(TSFileConfig.VALUE_BITS_LENGTH_32BIT - leadingZeroNumTmp -
tailingZeroNumTmp,
+ out, TSFileConfig.FLOAT_VALUE_LENGTH - 1, 0);
+ writeBits(tmp, out, TSFileConfig.VALUE_BITS_LENGTH_32BIT - 1 -
leadingZeroNumTmp,
+ tailingZeroNumTmp);
}
}
preValue = nextValue;
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/SinglePrecisionEncoderV2.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/SinglePrecisionEncoderV2.java
new file mode 100644
index 0000000..58940b4
--- /dev/null
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/SinglePrecisionEncoderV2.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.encoder;
+
+import static
org.apache.iotdb.tsfile.common.conf.TSFileConfig.GORILLA_ENCODING_ENDING_FLOAT;
+
+import java.io.ByteArrayOutputStream;
+
+/**
+ * <p> This class includes code modified from Michael Burman's gorilla-tsc
project.
+ *
+ * <p> Copyright: 2016-2018 Michael Burman and/or other contributors
+ * <p> Project page: https://github.com/burmanm/gorilla-tsc
+ * <p> License: http://www.apache.org/licenses/LICENSE-2.0
+ */
+public class SinglePrecisionEncoderV2 extends IntGorillaEncoder {
+
+ @Override
+ public final void encode(float value, ByteArrayOutputStream out) {
+ encode(Float.floatToRawIntBits(value), out);
+ }
+
+ @Override
+ public void flush(ByteArrayOutputStream out) {
+ // ending stream
+ encode(GORILLA_ENCODING_ENDING_FLOAT, out);
+
+ // flip the byte no matter it is empty or not
+ // the empty ending byte is necessary when decoding
+ bitsLeft = 0;
+ flipByte(out);
+
+ // the encoder may be reused, so let us reset it
+ reset();
+ }
+}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java
index 05fb304..0f6cd7c 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java
@@ -57,15 +57,17 @@ public abstract class TSEncodingBuilder {
public static TSEncodingBuilder getEncodingBuilder(TSEncoding type) {
switch (type) {
case PLAIN:
- return new PLAIN();
+ return new Plain();
case RLE:
- return new RLE();
+ return new Rle();
case TS_2DIFF:
- return new TS_2DIFF();
- case GORILLA:
- return new GORILLA();
+ return new Ts2Diff();
+ case GORILLA_V1:
+ return new GorillaV1();
case REGULAR:
- return new REGULAR();
+ return new Regular();
+ case GORILLA:
+ return new GorillaV2();
default:
throw new UnsupportedOperationException(type.toString());
}
@@ -97,7 +99,7 @@ public abstract class TSEncodingBuilder {
/**
* for all TSDataType.
*/
- public static class PLAIN extends TSEncodingBuilder {
+ public static class Plain extends TSEncodingBuilder {
private int maxStringLength =
TSFileDescriptor.getInstance().getConfig().getMaxStringLength();
@@ -126,7 +128,7 @@ public abstract class TSEncodingBuilder {
/**
* for ENUMS, INT32, BOOLEAN, INT64, FLOAT, DOUBLE.
*/
- public static class RLE extends TSEncodingBuilder {
+ public static class Rle extends TSEncodingBuilder {
private int maxPointNumber =
TSFileDescriptor.getInstance().getConfig().getFloatPrecision();
@@ -175,7 +177,7 @@ public abstract class TSEncodingBuilder {
/**
* for INT32, INT64, FLOAT, DOUBLE.
*/
- public static class TS_2DIFF extends TSEncodingBuilder {
+ public static class Ts2Diff extends TSEncodingBuilder {
private int maxPointNumber = 0;
@@ -218,23 +220,22 @@ public abstract class TSEncodingBuilder {
public String toString() {
return JsonFormatConstant.MAX_POINT_NUMBER + ":" + maxPointNumber;
}
-
}
/**
- * for ENUMS.
+ * for FLOAT, DOUBLE.
*/
- public static class GORILLA extends TSEncodingBuilder {
+ public static class GorillaV1 extends TSEncodingBuilder {
@Override
public Encoder getEncoder(TSDataType type) {
switch (type) {
case FLOAT:
- return new SinglePrecisionEncoder();
+ return new SinglePrecisionEncoderV1();
case DOUBLE:
- return new DoublePrecisionEncoder();
+ return new DoublePrecisionEncoderV1();
default:
- throw new UnSupportedDataTypeException("GORILLA doesn't support data
type: " + type);
+ throw new UnSupportedDataTypeException("GORILLA_V1 doesn't support
data type: " + type);
}
}
@@ -242,13 +243,12 @@ public abstract class TSEncodingBuilder {
public void initFromProps(Map<String, String> props) {
// allowed do nothing
}
-
}
/**
* for INT32, INT64
*/
- public static class REGULAR extends TSEncodingBuilder {
+ public static class Regular extends TSEncodingBuilder {
@Override
public Encoder getEncoder(TSDataType type) {
@@ -267,4 +267,31 @@ public abstract class TSEncodingBuilder {
// allowed do nothing
}
}
+
+ /**
+ * for FLOAT, DOUBLE, INT, LONG.
+ */
+ public static class GorillaV2 extends TSEncodingBuilder {
+
+ @Override
+ public Encoder getEncoder(TSDataType type) {
+ switch (type) {
+ case FLOAT:
+ return new SinglePrecisionEncoderV2();
+ case DOUBLE:
+ return new DoublePrecisionEncoderV2();
+ case INT32:
+ return new IntGorillaEncoder();
+ case INT64:
+ return new LongGorillaEncoder();
+ default:
+ throw new UnSupportedDataTypeException("GORILLA doesn't support data
type: " + type);
+ }
+ }
+
+ @Override
+ public void initFromProps(Map<String, String> props) {
+ // allowed do nothing
+ }
+ }
}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/TSEncoding.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/TSEncoding.java
index af54841..a909e47 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/TSEncoding.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/TSEncoding.java
@@ -20,7 +20,7 @@ package org.apache.iotdb.tsfile.file.metadata.enums;
public enum TSEncoding {
- PLAIN, PLAIN_DICTIONARY, RLE, DIFF, TS_2DIFF, BITMAP, GORILLA, REGULAR;
+ PLAIN, PLAIN_DICTIONARY, RLE, DIFF, TS_2DIFF, BITMAP, GORILLA_V1, REGULAR,
GORILLA;
/**
* judge the encoding deserialize type.
@@ -33,14 +33,14 @@ public enum TSEncoding {
}
public static byte deserializeToByte(short encoding) {
- if (encoding >= 8 || encoding < 0) {
+ if (encoding < 0 || 8 < encoding) {
throw new IllegalArgumentException("Invalid input: " + encoding);
}
return (byte) encoding;
}
private static TSEncoding getTsEncoding(short encoding) {
- if (encoding >= 8 || encoding < 0) {
+ if (encoding < 0 || 8 < encoding) {
throw new IllegalArgumentException("Invalid input: " + encoding);
}
switch (encoding) {
@@ -55,9 +55,11 @@ public enum TSEncoding {
case 5:
return BITMAP;
case 6:
- return GORILLA;
+ return GORILLA_V1;
case 7:
return REGULAR;
+ case 8:
+ return GORILLA;
default:
return PLAIN;
}
@@ -101,10 +103,12 @@ public enum TSEncoding {
return 4;
case BITMAP:
return 5;
- case GORILLA:
+ case GORILLA_V1:
return 6;
case REGULAR:
return 7;
+ case GORILLA:
+ return 8;
default:
return 0;
}
diff --git
a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/GorillaDecoderTest.java
b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/GorillaDecoderV1Test.java
similarity index 89%
rename from
tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/GorillaDecoderTest.java
rename to
tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/GorillaDecoderV1Test.java
index 071b3e3..8a34d22 100644
---
a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/GorillaDecoderTest.java
+++
b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/GorillaDecoderV1Test.java
@@ -33,15 +33,13 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.iotdb.tsfile.encoding.decoder.DoublePrecisionDecoder;
-import org.apache.iotdb.tsfile.encoding.decoder.SinglePrecisionDecoder;
-import org.apache.iotdb.tsfile.encoding.encoder.DoublePrecisionEncoder;
+import org.apache.iotdb.tsfile.encoding.encoder.DoublePrecisionEncoderV1;
import org.apache.iotdb.tsfile.encoding.encoder.Encoder;
-import org.apache.iotdb.tsfile.encoding.encoder.SinglePrecisionEncoder;
+import org.apache.iotdb.tsfile.encoding.encoder.SinglePrecisionEncoderV1;
-public class GorillaDecoderTest {
+public class GorillaDecoderV1Test {
- private static final Logger logger =
LoggerFactory.getLogger(GorillaDecoderTest.class);
+ private static final Logger logger =
LoggerFactory.getLogger(GorillaDecoderV1Test.class);
private final double delta = 0.0000001;
private final int floatMaxPointValue = 10000;
private final long doubleMaxPointValue = 1000000000000000L;
@@ -90,7 +88,7 @@ public class GorillaDecoderTest {
@Test
public void testNegativeNumber() throws IOException {
- Encoder encoder = new SinglePrecisionEncoder();
+ Encoder encoder = new SinglePrecisionEncoderV1();
ByteArrayOutputStream baos = new ByteArrayOutputStream();
float value = -7.101f;
encoder.encode(value, baos);
@@ -103,7 +101,7 @@ public class GorillaDecoderTest {
encoder.flush(baos);
ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
for (int i = 0; i < 2; i++) {
- Decoder decoder = new SinglePrecisionDecoder();
+ Decoder decoder = new SinglePrecisionDecoderV1();
if (decoder.hasNext(buffer)) {
assertEquals(value, decoder.readFloat(buffer), delta);
}
@@ -118,7 +116,7 @@ public class GorillaDecoderTest {
@Test
public void testZeroNumber() throws IOException {
- Encoder encoder = new DoublePrecisionEncoder();
+ Encoder encoder = new DoublePrecisionEncoderV1();
ByteArrayOutputStream baos = new ByteArrayOutputStream();
double value = 0f;
encoder.encode(value, baos);
@@ -131,7 +129,7 @@ public class GorillaDecoderTest {
encoder.flush(baos);
ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
for (int i = 0; i < 2; i++) {
- Decoder decoder = new DoublePrecisionDecoder();
+ Decoder decoder = new DoublePrecisionDecoderV1();
if (decoder.hasNext(buffer)) {
assertEquals(value, decoder.readDouble(buffer), delta);
}
@@ -160,7 +158,7 @@ public class GorillaDecoderTest {
@Test
public void testFloat() throws IOException {
- Encoder encoder = new SinglePrecisionEncoder();
+ Encoder encoder = new SinglePrecisionEncoderV1();
ByteArrayOutputStream baos = new ByteArrayOutputStream();
float value = 7.101f;
int num = 10000;
@@ -169,7 +167,7 @@ public class GorillaDecoderTest {
}
encoder.flush(baos);
ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
- Decoder decoder = new SinglePrecisionDecoder();
+ Decoder decoder = new SinglePrecisionDecoderV1();
for (int i = 0; i < num; i++) {
if (decoder.hasNext(buffer)) {
assertEquals(value + 2 * i, decoder.readFloat(buffer), delta);
@@ -181,7 +179,7 @@ public class GorillaDecoderTest {
@Test
public void testDouble() throws IOException {
- Encoder encoder = new DoublePrecisionEncoder();
+ Encoder encoder = new DoublePrecisionEncoderV1();
ByteArrayOutputStream baos = new ByteArrayOutputStream();
double value = 7.101f;
int num = 1000;
@@ -190,7 +188,7 @@ public class GorillaDecoderTest {
}
encoder.flush(baos);
ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
- Decoder decoder = new DoublePrecisionDecoder();
+ Decoder decoder = new DoublePrecisionDecoderV1();
for (int i = 0; i < num; i++) {
if (decoder.hasNext(buffer)) {
// System.out.println("turn "+i);
@@ -203,7 +201,7 @@ public class GorillaDecoderTest {
private void testFloatLength(List<Float> valueList, boolean isDebug, int
repeatCount)
throws Exception {
- Encoder encoder = new SinglePrecisionEncoder();
+ Encoder encoder = new SinglePrecisionEncoderV1();
ByteArrayOutputStream baos = new ByteArrayOutputStream();
for (int i = 0; i < repeatCount; i++) {
for (float value : valueList) {
@@ -214,7 +212,7 @@ public class GorillaDecoderTest {
ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
for (int i = 0; i < repeatCount; i++) {
- Decoder decoder = new SinglePrecisionDecoder();
+ Decoder decoder = new SinglePrecisionDecoderV1();
for (float value : valueList) {
// System.out.println("Repeat: "+i+" value: "+value);
if (decoder.hasNext(buffer)) {
@@ -232,7 +230,7 @@ public class GorillaDecoderTest {
private void testDoubleLength(List<Double> valueList, boolean isDebug, int
repeatCount)
throws Exception {
- Encoder encoder = new DoublePrecisionEncoder();
+ Encoder encoder = new DoublePrecisionEncoderV1();
ByteArrayOutputStream baos = new ByteArrayOutputStream();
for (int i = 0; i < repeatCount; i++) {
for (double value : valueList) {
@@ -244,7 +242,7 @@ public class GorillaDecoderTest {
ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
for (int i = 0; i < repeatCount; i++) {
- Decoder decoder = new DoublePrecisionDecoder();
+ Decoder decoder = new DoublePrecisionDecoderV1();
for (double value : valueList) {
if (decoder.hasNext(buffer)) {
double value_ = decoder.readDouble(buffer);
diff --git
a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/GorillaDecoderV2Test.java
b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/GorillaDecoderV2Test.java
new file mode 100644
index 0000000..2a5c16b
--- /dev/null
+++
b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/GorillaDecoderV2Test.java
@@ -0,0 +1,592 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.decoder;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.iotdb.tsfile.encoding.encoder.DoublePrecisionEncoderV2;
+import org.apache.iotdb.tsfile.encoding.encoder.Encoder;
+import org.apache.iotdb.tsfile.encoding.encoder.IntGorillaEncoder;
+import org.apache.iotdb.tsfile.encoding.encoder.LongGorillaEncoder;
+import org.apache.iotdb.tsfile.encoding.encoder.SinglePrecisionEncoderV2;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class GorillaDecoderV2Test {
+
+ private static final double DELTA = 0;
+ private static final int FLOAT_MAX_POINT_VALUE = 10000;
+ private static final long DOUBLE_MAX_POINT_VALUE = 1000000000000000L;
+
+ private static final List<Float> floatList = new ArrayList<>();
+ private static final List<Double> doubleList = new ArrayList<>();
+ private static final List<Integer> intList = new ArrayList<>();
+ private static final List<Long> longList = new ArrayList<>();
+
+ private static final List<Integer> iterations = new ArrayList<>();
+
+ @BeforeClass
+ public static void setUp() {
+ int hybridCount = 11;
+ int hybridNum = 50;
+ int hybridStart = 2000;
+ for (int i = 0; i < hybridNum; i++) {
+ for (int j = 0; j < hybridCount; j++) {
+ floatList.add((float) hybridStart / FLOAT_MAX_POINT_VALUE);
+ doubleList.add((double) hybridStart / DOUBLE_MAX_POINT_VALUE);
+ intList.add(hybridStart);
+ longList.add((long) hybridStart);
+ }
+ for (int j = 0; j < hybridCount; j++) {
+ floatList.add((float) hybridStart / FLOAT_MAX_POINT_VALUE);
+ doubleList.add((double) hybridStart / DOUBLE_MAX_POINT_VALUE);
+ intList.add(hybridStart);
+ longList.add((long) hybridStart);
+ hybridStart += 3;
+ }
+ hybridCount += 2;
+ }
+
+ iterations.add(1);
+ iterations.add(3);
+ iterations.add(8);
+ iterations.add(16);
+ iterations.add(1000);
+ iterations.add(10000);
+ }
+
+ @Test
+ public void testIntSingleValue() throws IOException {
+ Encoder encoder = new IntGorillaEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ encoder.encode(777, baos);
+ encoder.flush(baos);
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ Decoder decoder = new IntGorillaDecoder();
+ if (decoder.hasNext(buffer)) {
+ assertEquals(777, decoder.readInt(buffer));
+ }
+ if (decoder.hasNext(buffer)) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testFloatSingleValue() throws IOException {
+ Encoder encoder = new SinglePrecisionEncoderV2();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ encoder.encode(Float.MAX_VALUE, baos);
+ encoder.flush(baos);
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ Decoder decoder = new SinglePrecisionDecoderV2();
+ if (decoder.hasNext(buffer)) {
+ assertEquals(Float.MAX_VALUE, decoder.readFloat(buffer), DELTA);
+ }
+ if (decoder.hasNext(buffer)) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testLongSingleValue() throws IOException {
+ Encoder encoder = new LongGorillaEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ encoder.encode((long) Integer.MAX_VALUE + 10, baos);
+ encoder.flush(baos);
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ Decoder decoder = new LongGorillaDecoder();
+ if (decoder.hasNext(buffer)) {
+ assertEquals((long) Integer.MAX_VALUE + 10, decoder.readLong(buffer));
+ }
+ if (decoder.hasNext(buffer)) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testDoubleSingleValue() throws IOException {
+ Encoder encoder = new DoublePrecisionEncoderV2();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ encoder.encode(Double.MAX_VALUE, baos);
+ encoder.flush(baos);
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ Decoder decoder = new DoublePrecisionDecoderV2();
+ if (decoder.hasNext(buffer)) {
+ assertEquals(Double.MAX_VALUE, decoder.readDouble(buffer), DELTA);
+ }
+ if (decoder.hasNext(buffer)) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testIntZeroNumber() throws IOException {
+ Encoder encoder = new IntGorillaEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ int value = 0;
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.flush(baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.flush(baos);
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+ for (int i = 0; i < 2; i++) {
+ Decoder decoder = new IntGorillaDecoder();
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readInt(buffer), DELTA);
+ }
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readInt(buffer), DELTA);
+ }
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readInt(buffer), DELTA);
+ }
+ }
+ }
+
+ @Test
+ public void testFloatZeroNumber() throws IOException {
+ Encoder encoder = new SinglePrecisionEncoderV2();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ float value = 0f;
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.flush(baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.flush(baos);
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+ for (int i = 0; i < 2; i++) {
+ Decoder decoder = new SinglePrecisionDecoderV2();
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readFloat(buffer), DELTA);
+ }
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readFloat(buffer), DELTA);
+ }
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readFloat(buffer), DELTA);
+ }
+ }
+ }
+
+ @Test
+ public void testLongZeroNumber() throws IOException {
+ Encoder encoder = new LongGorillaEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ long value = 0;
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.flush(baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.flush(baos);
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+ for (int i = 0; i < 2; i++) {
+ Decoder decoder = new LongGorillaDecoder();
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readLong(buffer), DELTA);
+ }
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readLong(buffer), DELTA);
+ }
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readLong(buffer), DELTA);
+ }
+ }
+ }
+
+ @Test
+ public void testDoubleZeroNumber() throws IOException {
+ Encoder encoder = new DoublePrecisionEncoderV2();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ double value = 0f;
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.flush(baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.encode(value, baos);
+ encoder.flush(baos);
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+ for (int i = 0; i < 2; i++) {
+ Decoder decoder = new DoublePrecisionDecoderV2();
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readDouble(buffer), DELTA);
+ }
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readDouble(buffer), DELTA);
+ }
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value, decoder.readDouble(buffer), DELTA);
+ }
+ }
+ }
+
+ @Test
+ public void testInteger() throws IOException {
+ for (Integer num : iterations) {
+ Encoder encoder = new IntGorillaEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ int value = 7;
+ for (int i = 0; i < num; i++) {
+ encoder.encode(value + 2 * i, baos);
+ }
+ encoder.flush(baos);
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ Decoder decoder = new IntGorillaDecoder();
+ for (int i = 0; i < num; i++) {
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value + 2 * i, decoder.readInt(buffer));
+ continue;
+ }
+ fail();
+ }
+ if (decoder.hasNext(buffer)) {
+ fail();
+ }
+ }
+ }
+
+ @Test
+ public void testFloat() throws IOException {
+ for (Integer num : iterations) {
+ Encoder encoder = new SinglePrecisionEncoderV2();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ float value = 7.101f;
+ for (int i = 0; i < num; i++) {
+ encoder.encode(value + 2 * i, baos);
+ }
+ encoder.flush(baos);
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ Decoder decoder = new SinglePrecisionDecoderV2();
+ for (int i = 0; i < num; i++) {
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value + 2 * i, decoder.readFloat(buffer), DELTA);
+ continue;
+ }
+ fail();
+ }
+ if (decoder.hasNext(buffer)) {
+ fail();
+ }
+ }
+ }
+
+ @Test
+ public void testLong() throws IOException {
+ for (Integer num : iterations) {
+ Encoder encoder = new LongGorillaEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ long value = 7;
+ for (int i = 0; i < num; i++) {
+ encoder.encode(value + 2 * i, baos);
+ }
+ encoder.flush(baos);
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ Decoder decoder = new LongGorillaDecoder();
+ for (int i = 0; i < num; i++) {
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value + 2 * i, decoder.readLong(buffer));
+ continue;
+ }
+ fail();
+ }
+ if (decoder.hasNext(buffer)) {
+ fail();
+ }
+ }
+ }
+
+ @Test
+ public void testDouble() throws IOException {
+ for (Integer num : iterations) {
+ Encoder encoder = new DoublePrecisionEncoderV2();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ double value = 7.101f;
+ for (int i = 0; i < num; i++) {
+ encoder.encode(value + 2 * i, baos);
+ }
+ encoder.flush(baos);
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ Decoder decoder = new DoublePrecisionDecoderV2();
+ for (int i = 0; i < num; i++) {
+ if (decoder.hasNext(buffer)) {
+ assertEquals(value + 2 * i, decoder.readDouble(buffer), DELTA);
+ continue;
+ }
+ fail();
+ }
+ if (decoder.hasNext(buffer)) {
+ fail();
+ }
+ }
+ }
+
+ @Test
+ public void testIntegerRepeat() throws Exception {
+ for (int i = 1; i <= 10; i++) {
+ testInteger(i);
+ }
+ }
+
+ private void testInteger(int repeatCount) throws Exception {
+ Encoder encoder = new IntGorillaEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ for (int i = 0; i < repeatCount; i++) {
+ for (int value : GorillaDecoderV2Test.intList) {
+ encoder.encode(value, baos);
+ }
+ encoder.flush(baos);
+ }
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ for (int i = 0; i < repeatCount; i++) {
+ Decoder decoder = new IntGorillaDecoder();
+ for (int expected : GorillaDecoderV2Test.intList) {
+ if (decoder.hasNext(buffer)) {
+ int actual = decoder.readInt(buffer);
+ assertEquals(expected, actual);
+ continue;
+ }
+ fail();
+ }
+ }
+
+ encoder = new IntGorillaEncoder();
+ baos = new ByteArrayOutputStream();
+ for (int i = 0; i < repeatCount; i++) {
+ for (int value : GorillaDecoderV2Test.intList) {
+ encoder.encode(-value, baos);
+ }
+ encoder.flush(baos);
+ }
+
+ buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ for (int i = 0; i < repeatCount; i++) {
+ Decoder decoder = new IntGorillaDecoder();
+ for (int expected : GorillaDecoderV2Test.intList) {
+ if (decoder.hasNext(buffer)) {
+ int actual = decoder.readInt(buffer);
+ assertEquals(expected, -actual);
+ continue;
+ }
+ fail();
+ }
+ }
+ }
+
+ @Test
+ public void testFloatRepeat() throws Exception {
+ for (int i = 1; i <= 10; i++) {
+ testFloat(i);
+ }
+ }
+
+ private void testFloat(int repeatCount) throws Exception {
+ Encoder encoder = new SinglePrecisionEncoderV2();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ for (int i = 0; i < repeatCount; i++) {
+ for (float value : GorillaDecoderV2Test.floatList) {
+ encoder.encode(value, baos);
+ }
+ encoder.flush(baos);
+ }
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ for (int i = 0; i < repeatCount; i++) {
+ Decoder decoder = new SinglePrecisionDecoderV2();
+ for (float expected : GorillaDecoderV2Test.floatList) {
+ if (decoder.hasNext(buffer)) {
+ float actual = decoder.readFloat(buffer);
+ assertEquals(expected, actual, DELTA);
+ continue;
+ }
+ fail();
+ }
+ }
+
+ encoder = new SinglePrecisionEncoderV2();
+ baos = new ByteArrayOutputStream();
+ for (int i = 0; i < repeatCount; i++) {
+ for (float value : GorillaDecoderV2Test.floatList) {
+ encoder.encode(-value, baos);
+ }
+ encoder.flush(baos);
+ }
+
+ buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ for (int i = 0; i < repeatCount; i++) {
+ Decoder decoder = new SinglePrecisionDecoderV2();
+ for (float expected : GorillaDecoderV2Test.floatList) {
+ if (decoder.hasNext(buffer)) {
+ float actual = decoder.readFloat(buffer);
+ assertEquals(expected, -actual, DELTA);
+ continue;
+ }
+ fail();
+ }
+ }
+ }
+
+ @Test
+ public void testLongRepeat() throws Exception {
+ for (int i = 1; i <= 10; i++) {
+ testLong(i);
+ }
+ }
+
+ private void testLong(int repeatCount) throws Exception {
+ Encoder encoder = new LongGorillaEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ for (int i = 0; i < repeatCount; i++) {
+ for (long value : GorillaDecoderV2Test.longList) {
+ encoder.encode(value, baos);
+ }
+ encoder.flush(baos);
+ }
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ for (int i = 0; i < repeatCount; i++) {
+ Decoder decoder = new LongGorillaDecoder();
+ for (long expected : GorillaDecoderV2Test.longList) {
+ if (decoder.hasNext(buffer)) {
+ long actual = decoder.readLong(buffer);
+ assertEquals(expected, actual);
+ continue;
+ }
+ fail();
+ }
+ }
+
+ encoder = new LongGorillaEncoder();
+ baos = new ByteArrayOutputStream();
+ for (int i = 0; i < repeatCount; i++) {
+ for (long value : GorillaDecoderV2Test.longList) {
+ encoder.encode(-value, baos);
+ }
+ encoder.flush(baos);
+ }
+
+ buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ for (int i = 0; i < repeatCount; i++) {
+ Decoder decoder = new LongGorillaDecoder();
+ for (long expected : GorillaDecoderV2Test.longList) {
+ if (decoder.hasNext(buffer)) {
+ long actual = decoder.readLong(buffer);
+ assertEquals(expected, -actual);
+ continue;
+ }
+ fail();
+ }
+ }
+ }
+
+ @Test
+ public void testDoubleRepeat() throws Exception {
+ for (int i = 1; i <= 10; i++) {
+ testDouble(i);
+ }
+ }
+
+ private void testDouble(int repeatCount) throws Exception {
+ Encoder encoder = new DoublePrecisionEncoderV2();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ for (int i = 0; i < repeatCount; i++) {
+ for (double value : GorillaDecoderV2Test.doubleList) {
+ encoder.encode(value, baos);
+ }
+ encoder.flush(baos);
+ }
+
+ ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ for (int i = 0; i < repeatCount; i++) {
+ Decoder decoder = new DoublePrecisionDecoderV2();
+ for (double expected : GorillaDecoderV2Test.doubleList) {
+ if (decoder.hasNext(buffer)) {
+ double actual = decoder.readDouble(buffer);
+ assertEquals(expected, actual, DELTA);
+ continue;
+ }
+ fail();
+ }
+ }
+
+ encoder = new DoublePrecisionEncoderV2();
+ baos = new ByteArrayOutputStream();
+ for (int i = 0; i < repeatCount; i++) {
+ for (double value : GorillaDecoderV2Test.doubleList) {
+ encoder.encode(-value, baos);
+ }
+ encoder.flush(baos);
+ }
+
+ buffer = ByteBuffer.wrap(baos.toByteArray());
+
+ for (int i = 0; i < repeatCount; i++) {
+ Decoder decoder = new DoublePrecisionDecoderV2();
+ for (double expected : GorillaDecoderV2Test.doubleList) {
+ if (decoder.hasNext(buffer)) {
+ double actual = decoder.readDouble(buffer);
+ assertEquals(expected, -actual, DELTA);
+ continue;
+ }
+ fail();
+ }
+ }
+ }
+}
diff --git
a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/reader/PageReaderTest.java
b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/reader/PageReaderTest.java
index 1d1325d..65d86e6 100644
---
a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/reader/PageReaderTest.java
+++
b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/reader/PageReaderTest.java
@@ -25,18 +25,18 @@ import java.util.List;
import org.apache.iotdb.tsfile.encoding.common.EndianType;
import org.apache.iotdb.tsfile.encoding.decoder.Decoder;
import org.apache.iotdb.tsfile.encoding.decoder.DeltaBinaryDecoder;
-import org.apache.iotdb.tsfile.encoding.decoder.DoublePrecisionDecoder;
+import org.apache.iotdb.tsfile.encoding.decoder.DoublePrecisionDecoderV1;
import org.apache.iotdb.tsfile.encoding.decoder.IntRleDecoder;
import org.apache.iotdb.tsfile.encoding.decoder.LongRleDecoder;
import org.apache.iotdb.tsfile.encoding.decoder.PlainDecoder;
-import org.apache.iotdb.tsfile.encoding.decoder.SinglePrecisionDecoder;
+import org.apache.iotdb.tsfile.encoding.decoder.SinglePrecisionDecoderV1;
import org.apache.iotdb.tsfile.encoding.encoder.DeltaBinaryEncoder;
-import org.apache.iotdb.tsfile.encoding.encoder.DoublePrecisionEncoder;
+import org.apache.iotdb.tsfile.encoding.encoder.DoublePrecisionEncoderV1;
import org.apache.iotdb.tsfile.encoding.encoder.Encoder;
import org.apache.iotdb.tsfile.encoding.encoder.IntRleEncoder;
import org.apache.iotdb.tsfile.encoding.encoder.LongRleEncoder;
import org.apache.iotdb.tsfile.encoding.encoder.PlainEncoder;
-import org.apache.iotdb.tsfile.encoding.encoder.SinglePrecisionEncoder;
+import org.apache.iotdb.tsfile.encoding.encoder.SinglePrecisionEncoderV1;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.common.BatchData;
import org.apache.iotdb.tsfile.read.common.TimeRange;
@@ -92,8 +92,8 @@ public class PageReaderTest {
@Test
public void testFloat() {
- LoopWriteReadTest test = new LoopWriteReadTest("Test FLOAT", new
SinglePrecisionEncoder(),
- new SinglePrecisionDecoder(), TSDataType.FLOAT,
POINTS_COUNT_IN_ONE_PAGE) {
+ LoopWriteReadTest test = new LoopWriteReadTest("Test FLOAT", new
SinglePrecisionEncoderV1(),
+ new SinglePrecisionDecoderV1(), TSDataType.FLOAT,
POINTS_COUNT_IN_ONE_PAGE) {
@Override
public Object generateValueByIndex(int i) {
return Float.valueOf(i) / 10 - Float.valueOf(i) / 100;
@@ -101,8 +101,8 @@ public class PageReaderTest {
};
test.test(TSDataType.FLOAT);
- LoopWriteReadTest test2 = new LoopWriteReadTest("Test FLOAT", new
SinglePrecisionEncoder(),
- new SinglePrecisionDecoder(), TSDataType.FLOAT,
POINTS_COUNT_IN_ONE_PAGE) {
+ LoopWriteReadTest test2 = new LoopWriteReadTest("Test FLOAT", new
SinglePrecisionEncoderV1(),
+ new SinglePrecisionDecoderV1(), TSDataType.FLOAT,
POINTS_COUNT_IN_ONE_PAGE) {
@Override
public Object generateValueByIndex(int i) {
return Float.valueOf(i) / 100 - Float.valueOf(i) / 10;
@@ -113,8 +113,8 @@ public class PageReaderTest {
@Test
public void testDouble() {
- LoopWriteReadTest test = new LoopWriteReadTest("Test Double", new
DoublePrecisionEncoder(),
- new DoublePrecisionDecoder(), TSDataType.DOUBLE,
POINTS_COUNT_IN_ONE_PAGE) {
+ LoopWriteReadTest test = new LoopWriteReadTest("Test Double", new
DoublePrecisionEncoderV1(),
+ new DoublePrecisionDecoderV1(), TSDataType.DOUBLE,
POINTS_COUNT_IN_ONE_PAGE) {
@Override
public Object generateValueByIndex(int i) {
return Double.valueOf(i) / 10 - Double.valueOf(i) / 100;
@@ -122,8 +122,8 @@ public class PageReaderTest {
};
test.test(TSDataType.DOUBLE);
- LoopWriteReadTest test2 = new LoopWriteReadTest("Test Double", new
DoublePrecisionEncoder(),
- new DoublePrecisionDecoder(), TSDataType.DOUBLE,
POINTS_COUNT_IN_ONE_PAGE) {
+ LoopWriteReadTest test2 = new LoopWriteReadTest("Test Double", new
DoublePrecisionEncoderV1(),
+ new DoublePrecisionDecoderV1(), TSDataType.DOUBLE,
POINTS_COUNT_IN_ONE_PAGE) {
@Override
public Object generateValueByIndex(int i) {
return Double.valueOf(i) / 1000 - Double.valueOf(i) / 100;
diff --git
a/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/TsFileGeneratorForTest.java
b/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/TsFileGeneratorForTest.java
index b74004a..ce96854 100755
---
a/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/TsFileGeneratorForTest.java
+++
b/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/TsFileGeneratorForTest.java
@@ -182,7 +182,7 @@ public class TsFileGeneratorForTest {
schema.registerTimeseries(new Path("d1", "s5"), new
MeasurementSchema("s5", TSDataType.BOOLEAN, TSEncoding.RLE));
schema.registerTimeseries(new Path("d1", "s6"), new
MeasurementSchema("s6", TSDataType.FLOAT, TSEncoding.RLE,
CompressionType.SNAPPY,
Collections.singletonMap(Encoder.MAX_POINT_NUMBER, "5")));
- schema.registerTimeseries(new Path("d1", "s7"), new
MeasurementSchema("s7", TSDataType.DOUBLE, TSEncoding.GORILLA));
+ schema.registerTimeseries(new Path("d1", "s7"), new
MeasurementSchema("s7", TSDataType.DOUBLE, TSEncoding.GORILLA_V1));
schema.registerTimeseries(new Path("d2", "s1"), new
MeasurementSchema("s1", TSDataType.INT32, TSEncoding.RLE));
schema.registerTimeseries(new Path("d2", "s2"), new
MeasurementSchema("s2", TSDataType.INT64, TSEncoding.PLAIN));
diff --git
a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/TsFileReadWriteTest.java
b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/TsFileReadWriteTest.java
index d029e5f..8f55918 100644
---
a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/TsFileReadWriteTest.java
+++
b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/TsFileReadWriteTest.java
@@ -25,6 +25,8 @@ import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -68,31 +70,68 @@ public class TsFileReadWriteTest {
f = new File(path);
if (f.exists()) {
assertTrue(f.delete());
- ;
}
}
@Test
public void intTest() throws IOException, WriteProcessException {
- writeDataByTSRecord(TSDataType.INT32, (i) -> new IntDataPoint("sensor_1",
(int) i), TSEncoding.RLE);
+ List<TSEncoding> encodings = Arrays
+ .asList(TSEncoding.PLAIN, TSEncoding.RLE, TSEncoding.TS_2DIFF,
TSEncoding.REGULAR,
+ TSEncoding.GORILLA);
+ for (TSEncoding encoding : encodings) {
+ intTest(encoding);
+ }
+ }
+
+ private void intTest(TSEncoding encoding) throws IOException,
WriteProcessException {
+ writeDataByTSRecord(TSDataType.INT32, (i) -> new IntDataPoint("sensor_1",
(int) i), encoding);
readData((i, field, delta) -> assertEquals(i, field.getIntV()));
}
@Test
public void longTest() throws IOException, WriteProcessException {
- writeDataByTSRecord(TSDataType.INT64, (i) -> new LongDataPoint("sensor_1",
i), TSEncoding.RLE);
+ List<TSEncoding> encodings = Arrays
+ .asList(TSEncoding.PLAIN, TSEncoding.RLE, TSEncoding.TS_2DIFF,
TSEncoding.REGULAR,
+ TSEncoding.GORILLA);
+ for (TSEncoding encoding : encodings) {
+ longTest(encoding);
+ }
+ }
+
+ public void longTest(TSEncoding encoding) throws IOException,
WriteProcessException {
+ writeDataByTSRecord(TSDataType.INT64, (i) -> new LongDataPoint("sensor_1",
i), encoding);
readData((i, field, delta) -> assertEquals(i, field.getLongV()));
}
@Test
public void floatTest() throws IOException, WriteProcessException {
- writeDataByTSRecord(TSDataType.FLOAT, (i) -> new
FloatDataPoint("sensor_1", (float) i), TSEncoding.RLE);
+ List<TSEncoding> encodings = Arrays
+ .asList(TSEncoding.PLAIN, TSEncoding.RLE, TSEncoding.TS_2DIFF,
TSEncoding.GORILLA_V1,
+ TSEncoding.GORILLA);
+ for (TSEncoding encoding : encodings) {
+ floatTest(encoding);
+ }
+ }
+
+ public void floatTest(TSEncoding encoding) throws IOException,
WriteProcessException {
+ writeDataByTSRecord(TSDataType.FLOAT, (i) -> new
FloatDataPoint("sensor_1", (float) i),
+ encoding);
readData((i, field, delta) -> assertEquals(i, field.getFloatV(), delta));
}
@Test
public void doubleTest() throws IOException, WriteProcessException {
- writeDataByTSRecord(TSDataType.DOUBLE, (i) -> new
DoubleDataPoint("sensor_1", (double) i), TSEncoding.RLE);
+ List<TSEncoding> encodings = Arrays
+ .asList(TSEncoding.PLAIN, TSEncoding.RLE, TSEncoding.TS_2DIFF,
TSEncoding.GORILLA_V1,
+ TSEncoding.GORILLA);
+ for (TSEncoding encoding : encodings) {
+ doubleTest(encoding);
+ }
+ }
+
+ public void doubleTest(TSEncoding encoding) throws IOException,
WriteProcessException {
+ writeDataByTSRecord(TSDataType.DOUBLE, (i) -> new
DoubleDataPoint("sensor_1", (double) i),
+ encoding);
readData((i, field, delta) -> assertEquals(i, field.getDoubleV(), delta));
}
@@ -135,12 +174,14 @@ public class TsFileReadWriteTest {
@Test
public void readMeasurementWithRegularEncodingTest() throws IOException,
WriteProcessException {
TSFileDescriptor.getInstance().getConfig().setTimeEncoder("REGULAR");
- writeDataByTSRecord(TSDataType.INT64, (i) -> new LongDataPoint("sensor_1",
i), TSEncoding.REGULAR);
+ writeDataByTSRecord(TSDataType.INT64, (i) -> new LongDataPoint("sensor_1",
i),
+ TSEncoding.REGULAR);
readData((i, field, delta) -> assertEquals(i, field.getLongV()));
TSFileDescriptor.getInstance().getConfig().setTimeEncoder("TS_2DIFF");
}
- private void writeDataByTSRecord(TSDataType dataType, DataPointProxy proxy,
TSEncoding encodingType)
+ private void writeDataByTSRecord(TSDataType dataType, DataPointProxy proxy,
+ TSEncoding encodingType)
throws IOException, WriteProcessException {
int floatCount = 1024 * 1024 * 13 + 1023;
// add measurements into file schema
