This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/research/M4-visualization by
this push:
new abdf5f1863 add
abdf5f1863 is described below
commit abdf5f1863feb940fb9c59f5b1c4ccc08d4eef0f
Author: Lei Rui <[email protected]>
AuthorDate: Tue Oct 11 12:52:18 2022 +0800
add
---
.../iotdb/session/MyBasicOperationTest4.java | 131 +++++++++++----------
.../session/MyRealDataTest1_WriteAndQuery.java | 2 +-
.../encoding/decoder/DeltaBinaryDecoder.java | 86 +++++---------
.../encoding/encoder/DeltaBinaryEncoder.java | 87 +++++++++++++-
.../iotdb/tsfile/read/reader/page/PageReader.java | 3 +-
5 files changed, 182 insertions(+), 127 deletions(-)
diff --git
a/session/src/test/java/org/apache/iotdb/session/MyBasicOperationTest4.java
b/session/src/test/java/org/apache/iotdb/session/MyBasicOperationTest4.java
index c41a1bc5ff..e55853a600 100644
--- a/session/src/test/java/org/apache/iotdb/session/MyBasicOperationTest4.java
+++ b/session/src/test/java/org/apache/iotdb/session/MyBasicOperationTest4.java
@@ -1,10 +1,11 @@
package org.apache.iotdb.session;
-import java.text.DecimalFormat;
-import java.util.Random;
import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics;
import org.junit.Assert;
+import java.text.DecimalFormat;
+import java.util.Random;
+
public class MyBasicOperationTest4 {
public static void main(String[] args) {
@@ -24,58 +25,58 @@ public class MyBasicOperationTest4 {
int packWidth = 10;
// test op1
-// System.out.println("op1");
-// long[] op1_v1 = new long[repeat];
-// long[] op1_v2 = new long[repeat];
-// long start = System.nanoTime();
-// for (int i = 0; i < repeat; i++) {
-// int mask = 0b10000000;
-// long v1 = 0;
-// long v2 = 0;
-// for (int k = 0; k < 8; k++) {
-// if (k < splitPos) {
-// v1 = v1 | (buf[i] & mask);
-// mask = mask >> 1;
-// } else {
-// v2 = v2 | (buf[i] & mask);
-// mask = mask >> 1;
-// }
-// }
-// op1_v1[i] = v1 >> 5;
-// op1_v2[i] = v2 << (packWidth - (8 - splitPos));
-// }
-// long elapsedTime = System.nanoTime() - start;
-// op1.addValue(elapsedTime / 1000.0);
+ // System.out.println("op1");
+ // long[] op1_v1 = new long[repeat];
+ // long[] op1_v2 = new long[repeat];
+ // long start = System.nanoTime();
+ // for (int i = 0; i < repeat; i++) {
+ // int mask = 0b10000000;
+ // long v1 = 0;
+ // long v2 = 0;
+ // for (int k = 0; k < 8; k++) {
+ // if (k < splitPos) {
+ // v1 = v1 | (buf[i] & mask);
+ // mask = mask >> 1;
+ // } else {
+ // v2 = v2 | (buf[i] & mask);
+ // mask = mask >> 1;
+ // }
+ // }
+ // op1_v1[i] = v1 >> 5;
+ // op1_v2[i] = v2 << (packWidth - (8 - splitPos));
+ // }
+ // long elapsedTime = System.nanoTime() - start;
+ // op1.addValue(elapsedTime / 1000.0);
-// System.out.println("op1");
-// long[] op1_v1 = new long[repeat];
-// long[] op1_v2 = new long[repeat];
-// long start = System.nanoTime();
-// for (int i = 0; i < repeat; i++) {
-// long v1 = 0;
-// long v2 = 0;
-// char[] v = Long.toBinaryString(buf[i] & 0xff).toCharArray();
-//// System.out.println(v);
-// for (int k = 8 - v.length; k < 8; k++) {
-// if (k < splitPos) {
-// v1 = v1 << 1;
-// v1 = v1 + Character.getNumericValue(v[k - 8 + v.length]);
-// } else {
-// v2 = v2 << 1;
-// v2 = v2 + Character.getNumericValue(v[k - 8 + v.length]);
-// }
-// }
-// op1_v1[i] = v1;
-// op1_v2[i] = v2 << (packWidth - (8 - splitPos));
-//// System.out.println(op1_v1[i]);
-//// System.out.println(op1_v2[i]);
-// }
-// long elapsedTime = System.nanoTime() - start;
-// op1.addValue(elapsedTime / 1000.0);
+ // System.out.println("op1");
+ // long[] op1_v1 = new long[repeat];
+ // long[] op1_v2 = new long[repeat];
+ // long start = System.nanoTime();
+ // for (int i = 0; i < repeat; i++) {
+ // long v1 = 0;
+ // long v2 = 0;
+ // char[] v = Long.toBinaryString(buf[i] & 0xff).toCharArray();
+ //// System.out.println(v);
+ // for (int k = 8 - v.length; k < 8; k++) {
+ // if (k < splitPos) {
+ // v1 = v1 << 1;
+ // v1 = v1 + Character.getNumericValue(v[k - 8 + v.length]);
+ // } else {
+ // v2 = v2 << 1;
+ // v2 = v2 + Character.getNumericValue(v[k - 8 + v.length]);
+ // }
+ // }
+ // op1_v1[i] = v1;
+ // op1_v2[i] = v2 << (packWidth - (8 - splitPos));
+ //// System.out.println(op1_v1[i]);
+ //// System.out.println(op1_v2[i]);
+ // }
+ // long elapsedTime = System.nanoTime() - start;
+ // op1.addValue(elapsedTime / 1000.0);
System.out.println("op1");
-// long[] op1_v1 = new long[repeat];
-// long[] op1_v2 = new long[repeat];
+ // long[] op1_v1 = new long[repeat];
+ // long[] op1_v2 = new long[repeat];
long sum1 = 0;
long start = System.nanoTime();
for (int i = 0; i < repeat; i++) {
@@ -91,10 +92,10 @@ public class MyBasicOperationTest4 {
v2 = v2 + (b << (7 - k));
}
}
-// op1_v1[i] = v1;
-// op1_v2[i] = v2 << (packWidth - (8 - splitPos));
-// System.out.println(op1_v1[i]);
-// System.out.println(op1_v2[i]);
+ // op1_v1[i] = v1;
+ // op1_v2[i] = v2 << (packWidth - (8 - splitPos));
+ // System.out.println(op1_v1[i]);
+ // System.out.println(op1_v2[i]);
sum1 += v1;
sum1 += v2 << (packWidth - (8 - splitPos));
}
@@ -103,28 +104,28 @@ public class MyBasicOperationTest4 {
// test op2
System.out.println("op2");
-// long[] op2_v1 = new long[repeat];
-// long[] op2_v2 = new long[repeat];
+ // long[] op2_v1 = new long[repeat];
+ // long[] op2_v2 = new long[repeat];
long sum2 = 0;
start = System.nanoTime();
for (int i = 0; i < repeat; i++) {
byte data = buf[i];
long v1 = 0;
long v2 = 0;
-// op2_v1[i] = v1 | ((data & mask1) >> 5);
-// op2_v2[i] = v2 | ((data & mask2) << (packWidth - (8 - splitPos)));
-// System.out.println(op2_v1[i]);
-// System.out.println(op2_v2[i]);
+ // op2_v1[i] = v1 | ((data & mask1) >> 5);
+ // op2_v2[i] = v2 | ((data & mask2) << (packWidth - (8 -
splitPos)));
+ // System.out.println(op2_v1[i]);
+ // System.out.println(op2_v2[i]);
sum2 += v1 | ((data & mask1) >> 5);
sum2 += v2 | ((data & mask2) << (packWidth - (8 - splitPos)));
}
elapsedTime = System.nanoTime() - start;
op2.addValue(elapsedTime / 1000.0);
-// for (int i = 0; i < repeat; i++) {
-// Assert.assertEquals(op1_v1[i], op2_v1[i]);
-// Assert.assertEquals(op1_v2[i], op2_v2[i]);
-// }
+ // for (int i = 0; i < repeat; i++) {
+ // Assert.assertEquals(op1_v1[i], op2_v1[i]);
+ // Assert.assertEquals(op1_v2[i], op2_v2[i]);
+ // }
Assert.assertEquals(sum1, sum2);
printStat(op1, "op1-iterateBits");
diff --git
a/session/src/test/java/org/apache/iotdb/session/MyRealDataTest1_WriteAndQuery.java
b/session/src/test/java/org/apache/iotdb/session/MyRealDataTest1_WriteAndQuery.java
index 60335adcfc..5e35237174 100644
---
a/session/src/test/java/org/apache/iotdb/session/MyRealDataTest1_WriteAndQuery.java
+++
b/session/src/test/java/org/apache/iotdb/session/MyRealDataTest1_WriteAndQuery.java
@@ -178,7 +178,7 @@ public class MyRealDataTest1_WriteAndQuery {
private static long range = total_time_length;
private static boolean enableRegularityTimeDecode = true;
private static long regularTimeInterval = 1000L;
- private static String approach = "mac"; // 选择查询执行算法: 1: MAC, 2: MOC, 3: CPV
+ private static String approach = "cpv"; // 选择查询执行算法: 1: MAC, 2: MOC, 3: CPV
// private static String device = "root.sg1";
// private static String measurement = "RcvTime";
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java
index ace85f5311..2e740e09d6 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java
@@ -220,21 +220,24 @@ public abstract class DeltaBinaryDecoder extends Decoder {
readIntTotalCount = packNum;
nextReadIndex = 0;
- encodingLength = ceil(packNum * packWidth);
- deltaBuf = new byte[encodingLength];
- buffer.get(deltaBuf);
- allocateDataArray();
-
if (enableRegularityTimeDecode) {
long newRegularDelta = regularTimeInterval - minDeltaBase;
if (packWidth == 0) {
// [CASE 1]
+ encodingLength = ceil(packNum * packWidth);
+ deltaBuf = new byte[encodingLength];
+ buffer.get(deltaBuf);
+ allocateDataArray();
for (int i = 0; i < packNum; i++) {
data[i] = previous + minDeltaBase; // v=0
previous = data[i];
}
} else if (newRegularDelta < 0 || newRegularDelta >= Math.pow(2,
packWidth)) {
// [CASE 2] no need to compare equality cause impossible
+ encodingLength = ceil(packNum * packWidth);
+ deltaBuf = new byte[encodingLength];
+ buffer.get(deltaBuf);
+ allocateDataArray();
for (int i = 0; i < packNum; i++) {
long v = BytesUtils.bytesToLong(deltaBuf, packWidth * i,
packWidth);
data[i] = previous + minDeltaBase + v;
@@ -242,7 +245,14 @@ public abstract class DeltaBinaryDecoder extends Decoder {
}
} else {
// [CASE 3]
- // preprocess to get fallWithinMasks and regularBytes
+ // read regularBytes and deltaBuf
+ byte[][] regularBytes = readRegularBytes(buffer);
+ encodingLength = ceil(packNum * packWidth);
+ deltaBuf = new byte[encodingLength];
+ buffer.get(deltaBuf);
+ allocateDataArray();
+
+ // get fallWithinMasks
int[] fallWithinMasks = null;
if (packWidth >= 8) {
fallWithinMasks = null;
@@ -255,54 +265,6 @@ public abstract class DeltaBinaryDecoder extends Decoder {
} catch (Exception ignored) {
}
}
- byte[][] regularBytes;
- if (allRegularBytes.containsKey(new Pair<>(newRegularDelta,
packWidth))) {
- regularBytes = allRegularBytes.get(new Pair<>(newRegularDelta,
packWidth));
- } else { // TODO consider if the following steps can be accelerated
by using bytes instead
- // of bitwise get and set
- regularBytes = new byte[8][]; // 8 relative positions.
relativePos->bytes
- for (int i = 0; i < 8; i++) {
- // i is the starting position in the byte from high to low bits
-
- int endPos = i + packWidth - 1; // starting from 0
- int byteNum = endPos / 8 + 1;
- byte[] byteArray = new byte[byteNum];
- if (newRegularDelta != 0) {
- // put bit-packed newRegularDelta starting at position i,
- // and pad the front and back with newRegularDeltas.
- // Otherwise if newRegularDelta=0, just leave byteArray as
initial zeros
-
- // 1. deal with padding the first byte
- for (int x = i - 1; x >= 0; x--) {
- // y is the position in the bit-packed newRegularDelta,
0->packWidth-1 from low to
- // high bits
- int y = (i - x - 1) % packWidth;
- // get the bit indicated by y pos
- int value = BytesUtils.getLongN(newRegularDelta, y);
- // put the bit indicated by y pos into regularBytes
- // setByte pos is from high to low starting from 0,
corresponding to x
- byteArray[0] = BytesUtils.setByteN(byteArray[0], x, value);
- }
-
- // 2. deal with putting newRegularDeltas
- BytesUtils.longToBytes(newRegularDelta, byteArray, i,
packWidth);
-
- // 3. deal with padding the last byte
- for (int x = endPos + 1; x < byteNum * 8; x++) {
- // y is the position in the bit-packed newRegularDelta,
0->packWidth-1 from low to
- // high bits
- int y = packWidth - 1 - (x - endPos - 1) % packWidth;
- // get the bit indicated by y pos
- int value = BytesUtils.getLongN(newRegularDelta, y);
- // put the bit indicated by y pos into regularBytes
- // setByte pos is from high to low starting from 0,
corresponding to x
- byteArray[byteNum - 1] =
BytesUtils.setByteN(byteArray[byteNum - 1], x, value);
- }
- }
- regularBytes[i] = byteArray;
- }
- allRegularBytes.put(new Pair<>(newRegularDelta, packWidth),
regularBytes);
- }
// Begin decoding each number in this pack
for (int i = 0; i < packNum; i++) {
@@ -341,6 +303,10 @@ public abstract class DeltaBinaryDecoder extends Decoder {
}
}
} else { // without regularity-aware decoding
+ encodingLength = ceil(packNum * packWidth);
+ deltaBuf = new byte[encodingLength];
+ buffer.get(deltaBuf);
+ allocateDataArray();
readPack();
}
@@ -369,6 +335,18 @@ public abstract class DeltaBinaryDecoder extends Decoder {
firstValue = ReadWriteIOUtils.readLong(buffer);
}
+ private byte[][] readRegularBytes(ByteBuffer buffer) {
+ byte[][] regularBytes = new byte[8][];
+ for (int i = 0; i < 8; i++) {
+ int byteArrayLength = ReadWriteIOUtils.readInt(buffer);
+ regularBytes[i] = new byte[byteArrayLength];
+ for (int j = 0; j < byteArrayLength; j++) {
+ regularBytes[i][j] = ReadWriteIOUtils.readByte(buffer);
+ }
+ }
+ return regularBytes;
+ }
+
@Override
protected void allocateDataArray() {
data = new long[packNum];
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DeltaBinaryEncoder.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DeltaBinaryEncoder.java
index f2d60704c6..4efe7244e2 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DeltaBinaryEncoder.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DeltaBinaryEncoder.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.tsfile.encoding.encoder;
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.utils.BytesUtils;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
@@ -65,6 +66,8 @@ public abstract class DeltaBinaryEncoder extends Encoder {
blockSize = size;
}
+ protected abstract void writeHeaderToBytes() throws IOException;
+
protected abstract void writeHeader() throws IOException;
protected abstract void writeValueToBytes(int i);
@@ -84,12 +87,6 @@ public abstract class DeltaBinaryEncoder extends Encoder {
out.write(encodingBlockBuffer, 0, encodingLength);
}
- private void writeHeaderToBytes() throws IOException {
- ReadWriteIOUtils.write(writeIndex, out);
- ReadWriteIOUtils.write(writeWidth, out);
- writeHeader();
- }
-
private void flushBlockBuffer(ByteArrayOutputStream out) throws IOException {
if (writeIndex == -1) {
return;
@@ -147,6 +144,13 @@ public abstract class DeltaBinaryEncoder extends Encoder {
reset();
}
+ @Override
+ protected void writeHeaderToBytes() throws IOException {
+ ReadWriteIOUtils.write(writeIndex, out);
+ ReadWriteIOUtils.write(writeWidth, out);
+ writeHeader();
+ }
+
@Override
protected int calculateBitWidthsForDeltaBlockBuffer() {
int width = 0;
@@ -263,6 +267,77 @@ public abstract class DeltaBinaryEncoder extends Encoder {
deltaBlockBuffer[writeIndex++] = delta;
}
+ @Override
+ protected void writeHeaderToBytes() throws IOException {
+ ReadWriteIOUtils.write(writeIndex, out);
+ ReadWriteIOUtils.write(writeWidth, out);
+ writeHeader();
+ // TODO write (relativePos->bytes) for regular new delta
+ if
(TSFileDescriptor.getInstance().getConfig().isEnableRegularityTimeDecode()) {
+ long newRegularDelta =
+
TSFileDescriptor.getInstance().getConfig().getRegularTimeInterval() -
minDeltaBase;
+ if (writeWidth > 0 && 0 <= newRegularDelta && newRegularDelta <
Math.pow(2, writeWidth)) {
+
+ // byte[][] regularBytes = new byte[8][]; // 8 relative
positions.
+ // relativePos->bytes
+ for (int i = 0; i < 8; i++) {
+ // i is the starting position in the byte from high to low bits
+
+ int endPos = i + writeWidth - 1; // starting from 0
+ int byteNum = endPos / 8 + 1;
+ byte[] byteArray = new byte[byteNum];
+ if (newRegularDelta != 0) {
+ // put bit-packed newRegularDelta starting at position i,
+ // and pad the front and back with newRegularDeltas.
+ // Otherwise if newRegularDelta=0, just leave byteArray as
initial zeros
+
+ // 1. deal with padding the first byte
+ for (int x = i - 1; x >= 0; x--) {
+ // y is the position in the bit-packed newRegularDelta,
0->packWidth-1 from low to
+ // high bits
+ int y = (i - x - 1) % writeWidth;
+ // get the bit indicated by y pos
+ int value = BytesUtils.getLongN(newRegularDelta, y);
+ // put the bit indicated by y pos into regularBytes
+ // setByte pos is from high to low starting from 0,
corresponding to x
+ byteArray[0] = BytesUtils.setByteN(byteArray[0], x, value);
+ }
+
+ // 2. deal with putting newRegularDeltas
+ BytesUtils.longToBytes(newRegularDelta, byteArray, i,
writeWidth);
+
+ // 3. deal with padding the last byte
+ for (int x = endPos + 1; x < byteNum * 8; x++) {
+ // y is the position in the bit-packed newRegularDelta,
0->packWidth-1 from low to
+ // high bits
+ int y = writeWidth - 1 - (x - endPos - 1) % writeWidth;
+ // get the bit indicated by y pos
+ int value = BytesUtils.getLongN(newRegularDelta, y);
+ // put the bit indicated by y pos into regularBytes
+ // setByte pos is from high to low starting from 0,
corresponding to x
+ byteArray[byteNum - 1] = BytesUtils.setByteN(byteArray[byteNum
- 1], x, value);
+ }
+ }
+ // regularBytes[i] = byteArray;
+ int num = byteArray.length;
+ ReadWriteIOUtils.write(num, out);
+ for (int j = 0; j < num; j++) {
+ ReadWriteIOUtils.write(byteArray[j], out);
+ }
+ }
+ // // TODO write out
+ // for (int i = 0; i < 8; i++) {
+ // byte[] byteArray = regularBytes[i];
+ // int num = byteArray.length;
+ // ReadWriteIOUtils.write(num, out);
+ // for (int j = 0; j < num; j++) {
+ // ReadWriteIOUtils.write(byteArray[j], out);
+ // }
+ // }
+ }
+ }
+ }
+
@Override
protected void reset() {
firstValue = 0L;
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
index 870994f939..98a8763e18 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
@@ -277,7 +277,8 @@ public class PageReader implements IPageReader {
switch (dataType) {
case BOOLEAN:
boolean aBoolean = valueDecoder.readBoolean(valueBuffer);
- if (!isDeleted(timestamp) && (filter == null ||
filter.satisfy(timestamp, aBoolean))) { // TODO:remove
+ if (!isDeleted(timestamp)
+ && (filter == null || filter.satisfy(timestamp, aBoolean))) { //
TODO:remove
pageData.putBoolean(timestamp, aBoolean);
}
break;
