This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/research/M4-visualization by
this push:
new 48ccce48e9 update
48ccce48e9 is described below
commit 48ccce48e918f6d3552d79db03b174b29cf20f4f
Author: Lei Rui <[email protected]>
AuthorDate: Fri Oct 7 21:45:30 2022 +0800
update
---
.../session/MySmallRealDataWriteQueryTest.java | 5 +-
.../encoding/decoder/DeltaBinaryDecoder.java | 93 +++++++++++++++++-----
.../encoding/encoder/DeltaBinaryEncoder.java | 8 +-
3 files changed, 78 insertions(+), 28 deletions(-)
diff --git
a/session/src/test/java/org/apache/iotdb/session/MySmallRealDataWriteQueryTest.java
b/session/src/test/java/org/apache/iotdb/session/MySmallRealDataWriteQueryTest.java
index f44590e4e8..195aff618f 100644
---
a/session/src/test/java/org/apache/iotdb/session/MySmallRealDataWriteQueryTest.java
+++
b/session/src/test/java/org/apache/iotdb/session/MySmallRealDataWriteQueryTest.java
@@ -56,10 +56,11 @@ public class MySmallRealDataWriteQueryTest {
private static int deleteLenPercentage = 0; // 0-100
每次删除的时间长度，用chunkAvgTimeLen的百分比表示
private static int timeIdx = 0; // 时间戳idx，从0开始
private static int valueIdx = 1; // 值idx，从0开始
- private static int w = 2;
+ private static int w = 3;
private static long range = total_time_length;
- private static boolean enableRegularityTimeDecode = true;
+ private static boolean enableRegularityTimeDecode = false;
private static long regularTimeInterval = 511996L;
+// private static long regularTimeInterval = 511997L;
private static String approach = "cpv"; // 选择查询执行算法: 1: MAC, 2: MOC, 3: CPV
@Before
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java
index e5b610286c..53e6103f19 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java
@@ -19,19 +19,19 @@
package org.apache.iotdb.tsfile.encoding.decoder;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.encoding.encoder.DeltaBinaryEncoder;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.utils.BytesUtils;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
/**
* This class is a decoder for decoding the byte array that encoded by {@code
- * DeltaBinaryEncoder}.DeltaBinaryDecoder just supports integer and long
values.<br>
- * .
+ * DeltaBinaryEncoder}.DeltaBinaryDecoder just supports integer and long
values.<br> .
*
* @see DeltaBinaryEncoder
*/
@@ -40,16 +40,24 @@ public abstract class DeltaBinaryDecoder extends Decoder {
protected long count = 0;
protected byte[] deltaBuf;
- /** the first value in one pack. */
+ /**
+ * the first value in one pack.
+ */
protected int readIntTotalCount = 0;
protected int nextReadIndex = 0;
- /** max bit length of all value in a pack. */
+ /**
+ * max bit length of all value in a pack.
+ */
protected int packWidth;
- /** data number in this pack. */
+ /**
+ * data number in this pack.
+ */
protected int packNum;
- /** how many bytes data takes after encoding. */
+ /**
+ * how many bytes data takes after encoding.
+ */
protected int encodingLength;
public DeltaBinaryDecoder() {
@@ -82,7 +90,9 @@ public abstract class DeltaBinaryDecoder extends Decoder {
private int firstValue;
private int[] data;
private int previous;
- /** minimum value for all difference. */
+ /**
+ * minimum value for all difference.
+ */
private int minDeltaBase;
public IntDeltaDecoder() {
@@ -166,7 +176,9 @@ public abstract class DeltaBinaryDecoder extends Decoder {
private long firstValue;
private long[] data;
private long previous;
- /** minimum value for all difference. */
+ /**
+ * minimum value for all difference.
+ */
private long minDeltaBase;
private boolean enableRegularityTimeDecode;
@@ -217,15 +229,47 @@ public abstract class DeltaBinaryDecoder extends Decoder {
allocateDataArray();
if (enableRegularityTimeDecode) {
- // TODO encode regular time interval
- long newDelta = regularTimeInterval - minDeltaBase;
- int bitWidthToByteNum =
- ceil(packWidth); // packWidth has been made to be a multiple of 8
when encoding
- encodedRegularTimeInterval = new byte[bitWidthToByteNum];
- BytesUtils.longToBytes(newDelta, encodedRegularTimeInterval, 0,
packWidth);
+ // preprocess the regular time interval
+ long newRegularDelta = regularTimeInterval - minDeltaBase;
+ Map<Integer, byte[]> regularBytes = new HashMap<>();
+ for (int i = 0; i < 8; i++) {
+ // i is the starting position in the byte from high to low bits
+ int endPos = i + packWidth - 1; // starting from 0
+ int byteNum = endPos / 8 + 1;
+ byte[] byteArray = new byte[byteNum];
+ // put bit-packed newRegularDelta starting at position i,
+ // and pad the front and back with newRegularDeltas
+
+ // 1. deal with padding the first byte
+ for (int x = i - 1; x >= 0; x--) {
+ // y is the position in the bit-packed newRegularDelta,
0->packWidth-1 from low to high bits
+ int y = (i - x - 1) % packWidth;
+ // get the bit indicated by y pos
+ int value = BytesUtils.getLongN(newRegularDelta, y);
+ // put the bit indicated by y pos into regularBytes
+ // setByte pos is from high to low starting from 0, corresponding
to x
+ byteArray[0] = BytesUtils.setByteN(byteArray[0], x, value);
+ }
+
+ // 2. deal with putting newRegularDeltas
+ BytesUtils.longToBytes(newRegularDelta, byteArray, i, packWidth);
+
+ // 3. deal with padding the last byte
+ for (int x = endPos + 1; x < byteNum * 8; x++) {
+ // y is the position in the bit-packed newRegularDelta,
0->packWidth-1 from low to high bits
+ int y = packWidth - 1 - (x - endPos - 1) % packWidth;
+ // get the bit indicated by y pos
+ int value = BytesUtils.getLongN(newRegularDelta, y);
+ // put the bit indicated by y pos into regularBytes
+ // setByte pos is from high to low starting from 0, corresponding
to x
+ byteArray[byteNum - 1] = BytesUtils.setByteN(byteArray[byteNum -
1], x, value);
+ }
+
+ regularBytes.put(i, byteArray);
+ }
for (int i = 0; i < packNum; i++) {
- // TODO (1) extract bits from deltaBuf,
+ // (1) extract bits from deltaBuf,
// (2) compare bits with encodedRegularTimeInterval,
// (3) equal to reuse, else to convert
@@ -233,20 +277,25 @@ public abstract class DeltaBinaryDecoder extends Decoder {
data[i] = previous + minDeltaBase; // v=0
} else {
boolean equal = true;
- int pos = i * bitWidthToByteNum;
- for (int j = 0; j < bitWidthToByteNum; j++) { // compare encoded
bytes
- byte regular = encodedRegularTimeInterval[j];
- byte data = deltaBuf[pos + j];
+ int posByteIdx = i * packWidth / 8;
+ int pos = i * packWidth % 8; // the starting position in the byte
from high to low bits
+ byte[] byteArray = regularBytes.get(pos);
+ for (int k = 0; k < byteArray.length; k++, posByteIdx++) {
+ byte regular = byteArray[k];
+ byte data = deltaBuf[posByteIdx];
if (regular != data) {
equal = false;
break;
}
}
+
if (equal) {
data[i] = previous + regularTimeInterval;
+// System.out.println("[RL]equals");
} else {
long v = BytesUtils.bytesToLong(deltaBuf, packWidth * i,
packWidth);
data[i] = previous + minDeltaBase + v;
+// System.out.println("[RL]no");
}
}
previous = data[i];
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DeltaBinaryEncoder.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DeltaBinaryEncoder.java
index c7c87584d4..abc4221649 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DeltaBinaryEncoder.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DeltaBinaryEncoder.java
@@ -103,10 +103,10 @@ public abstract class DeltaBinaryEncoder extends Encoder {
}
writeWidth = calculateBitWidthsForDeltaBlockBuffer();
- // TODO ceil writeWidth to be a multiple of 8
- if
(TSFileDescriptor.getInstance().getConfig().isEnableRegularityTimeDecode()) {
- writeWidth = 8 * (int) Math.ceil((double) (writeWidth) / 8.0);
- }
+// // TODO ceil writeWidth to be a multiple of 8
+// if
(TSFileDescriptor.getInstance().getConfig().isEnableRegularityTimeDecode()) {
+// writeWidth = 8 * (int) Math.ceil((double) (writeWidth) / 8.0);
+// }
writeHeaderToBytes();
writeDataWithMinWidth();
