[iotdb] branch master updated: [IOTDB-1152] optimize regular data size in traversing (#2667)

[haonan]

This is an automated email from the ASF dual-hosted git repository.

haonan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git


The following commit(s) were added to refs/heads/master by this push:
     new c84771b  [IOTDB-1152] optimize regular data size in traversing (#2667)
c84771b is described below

commit c84771ba7a7821c9161d14fb6771c4094a767118
Author: wangchao316 <66939405+wangchao...@users.noreply.github.com>
AuthorDate: Tue Feb 23 10:28:41 2021 +0800

    [IOTDB-1152] optimize regular data size in traversing (#2667)
    
    * IOTDB-1152 optimize regular data size in traversing
---
 .../iotdb/tsfile/encoding/encoder/RegularDataEncoder.java   |  8 ++++++--
 .../decoder/regular/RegularDataEncoderIntegerTest.java      | 13 +++++++++++++
 .../decoder/regular/RegularDataEncoderLongTest.java         | 13 +++++++++++++
 3 files changed, 32 insertions(+), 2 deletions(-)

diff --git 
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/RegularDataEncoder.java
 
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/RegularDataEncoder.java
index 9dcf82a..7422263 100644
--- 
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/RegularDataEncoder.java
+++ 
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/RegularDataEncoder.java
@@ -51,6 +51,8 @@ public abstract class RegularDataEncoder extends Encoder {
 
   protected int writeIndex = -1;
 
+  protected int dataTotal;
+
   /**
    * constructor of RegularDataEncoder.
    *
@@ -185,6 +187,7 @@ public abstract class RegularDataEncoder extends Encoder {
       }
       firstValue = data[0];
       if (isMissingPoint) {
+        dataTotal = writeIndex;
         newBlockSize = ((data[writeIndex - 1] - data[0]) / minDeltaBase) + 1;
         writeIndex = newBlockSize;
       }
@@ -220,7 +223,7 @@ public abstract class RegularDataEncoder extends Encoder {
       bitmap = new BitSet(newBlockSize);
       bitmap.flip(0, newBlockSize);
       int offset = 0;
-      for (int i = 1; i < missingPointData.length; i++) {
+      for (int i = 1; i < dataTotal; i++) {
         int delta = missingPointData[i] - missingPointData[i - 1];
         if (delta != minDeltaBase) {
           int missingPointNum = (int) (delta / minDeltaBase) - 1;
@@ -311,6 +314,7 @@ public abstract class RegularDataEncoder extends Encoder {
       }
       firstValue = data[0];
       if (isMissingPoint) {
+        dataTotal = writeIndex;
         newBlockSize = (int) (((data[writeIndex - 1] - data[0]) / 
minDeltaBase) + 1);
         writeIndex = newBlockSize;
       }
@@ -346,7 +350,7 @@ public abstract class RegularDataEncoder extends Encoder {
       bitmap = new BitSet(newBlockSize);
       bitmap.flip(0, newBlockSize);
       int offset = 0;
-      for (int i = 1; i < missingPointData.length; i++) {
+      for (int i = 1; i < dataTotal; i++) {
         long delta = missingPointData[i] - missingPointData[i - 1];
         if (delta != minDeltaBase) {
           int missingPointNum = (int) (delta / minDeltaBase) - 1;
diff --git 
a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/regular/RegularDataEncoderIntegerTest.java
 
b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/regular/RegularDataEncoderIntegerTest.java
index 3ea0484..97a7781 100644
--- 
a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/regular/RegularDataEncoderIntegerTest.java
+++ 
b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/regular/RegularDataEncoderIntegerTest.java
@@ -98,6 +98,19 @@ public class RegularDataEncoderIntegerTest {
     shouldReadAndWrite(data, ROW_NUM);
   }
 
+  @Test
+  public void testMissingPointsDataSize() throws IOException {
+    int[] originalData = new int[] {1000, 1100, 1200, 1300, 1500, 2000};
+    out = new ByteArrayOutputStream();
+    writeData(originalData, 6);
+    byte[] page = out.toByteArray();
+    buffer = ByteBuffer.wrap(page);
+    int i = 0;
+    while (regularDataDecoder.hasNext(buffer)) {
+      assertEquals(originalData[i++], regularDataDecoder.readInt(buffer));
+    }
+  }
+
   private int[] getMissingPointData(int dataSize, int missingPointInterval) {
     ROW_NUM = dataSize;
 
diff --git 
a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/regular/RegularDataEncoderLongTest.java
 
b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/regular/RegularDataEncoderLongTest.java
index e53774a..e968c2d 100644
--- 
a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/regular/RegularDataEncoderLongTest.java
+++ 
b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/regular/RegularDataEncoderLongTest.java
@@ -139,6 +139,19 @@ public class RegularDataEncoderLongTest {
     shouldReadAndWrite(data, ROW_NUM);
   }
 
+  @Test
+  public void testMissingPointsDataSize() throws IOException {
+    long[] originalData = new long[] {1000, 1100, 1200, 1300, 1500, 2000};
+    out = new ByteArrayOutputStream();
+    writeData(originalData, 6);
+    byte[] page = out.toByteArray();
+    buffer = ByteBuffer.wrap(page);
+    int i = 0;
+    while (regularDataDecoder.hasNext(buffer)) {
+      assertEquals(originalData[i++], regularDataDecoder.readLong(buffer));
+    }
+  }
+
   private long[] getMissingPointData(List<String> originalData, int 
missingPointInterval) {
     List<String> dates = originalData;