This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/research/M4-visualization by
this push:
new 9d67411af9 new
9d67411af9 is described below
commit 9d67411af9183c617eddd29ea5d641b5e308d609
Author: Lei Rui <[email protected]>
AuthorDate: Sat Oct 8 19:17:04 2022 +0800
new
---
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 2 +-
.../org/apache/iotdb/db/service/TSServiceImpl.java | 7 +-
session/pom.xml | 5 +
.../apache/iotdb/session/MyRealDataQueryTest.java | 536 +++++++++++++++++++++
.../iotdb/session/MyRealDataWriteQueryTest.java | 260 +++++++++-
.../session/MySmallRealDataWriteQueryTest.java | 31 +-
tsfile/pom.xml | 5 +
.../tsfile/common/constant/TsFileConstant.java | 19 +
.../encoding/decoder/DeltaBinaryDecoder.java | 168 +++++--
.../encoding/encoder/DeltaBinaryEncoder.java | 9 +-
10 files changed, 947 insertions(+), 95 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 725c38d571..f09167dc3e 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -920,7 +920,7 @@ public class IoTDBConfig {
this.enableMetricService = enableMetricService;
}
- void setDataDirs(String[] dataDirs) {
+ public void setDataDirs(String[] dataDirs) {
this.dataDirs = dataDirs;
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
index 728ccc5658..b68e34cdd8 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
@@ -123,6 +123,7 @@ import
org.apache.iotdb.service.rpc.thrift.TSRawDataQueryReq;
import org.apache.iotdb.service.rpc.thrift.TSSetSchemaTemplateReq;
import org.apache.iotdb.service.rpc.thrift.TSSetTimeZoneReq;
import org.apache.iotdb.service.rpc.thrift.TSStatus;
+import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
import
org.apache.iotdb.tsfile.exception.filter.QueryFilterOptimizationException;
import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
@@ -664,7 +665,11 @@ public class TSServiceImpl implements TSIService.Iface {
public TSExecuteFinishResp executeFinish() throws TException {
TSExecuteFinishResp ret = new TSExecuteFinishResp();
ret.setStatus(RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS));
- ret.setExecutionInfo(IOMonitor.print());
+ ret.setExecutionInfo(
+ IOMonitor.print()
+ + ". timeColumnTS2DIFFLoadBatchCost="
+ + TsFileConstant.timeColumnTS2DIFFLoadBatchCost.getSum()
+ + "us");
IOMonitor.finish();
return ret;
}
diff --git a/session/pom.xml b/session/pom.xml
index e8ee8175a1..433ccac121 100644
--- a/session/pom.xml
+++ b/session/pom.xml
@@ -66,6 +66,11 @@
</plugins>
</build>
<dependencies>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-math3</artifactId>
+ <version>3.6.1</version>
+ </dependency>
<dependency>
<groupId>org.apache.iotdb</groupId>
<artifactId>service-rpc</artifactId>
diff --git
a/session/src/test/java/org/apache/iotdb/session/MyRealDataQueryTest.java
b/session/src/test/java/org/apache/iotdb/session/MyRealDataQueryTest.java
new file mode 100644
index 0000000000..08fd63bf2c
--- /dev/null
+++ b/session/src/test/java/org/apache/iotdb/session/MyRealDataQueryTest.java
@@ -0,0 +1,536 @@
+package org.apache.iotdb.session;
+
+import org.apache.iotdb.db.conf.IoTDBConfig;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.rpc.IoTDBConnectionException;
+import org.apache.iotdb.rpc.StatementExecutionException;
+import org.apache.iotdb.session.SessionDataSet.DataIterator;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.text.DecimalFormat;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import static
org.apache.iotdb.tsfile.common.constant.TsFileConstant.allRegularBytesSize;
+import static
org.apache.iotdb.tsfile.common.constant.TsFileConstant.byteArrayLengthStatistics;
+import static
org.apache.iotdb.tsfile.common.constant.TsFileConstant.countForHitNewDeltas;
+import static
org.apache.iotdb.tsfile.common.constant.TsFileConstant.countForNotHitNewDeltas;
+import static
org.apache.iotdb.tsfile.common.constant.TsFileConstant.countForRegularEqual;
+import static
org.apache.iotdb.tsfile.common.constant.TsFileConstant.countForRegularNOTEqual;
+import static
org.apache.iotdb.tsfile.common.constant.TsFileConstant.countForRegularNewDeltas;
+import static
org.apache.iotdb.tsfile.common.constant.TsFileConstant.prepareAllRegulars;
+import static
org.apache.iotdb.tsfile.common.constant.TsFileConstant.timeColumnTS2DIFFLoadBatchCost;
+
+public class MyRealDataQueryTest {
+
+ private static final IoTDBConfig config =
IoTDBDescriptor.getInstance().getConfig();
+
+ // * (1) min_time(%s), max_time(%s), first_value(%s), last_value(%s),
min_value(%s), max_value(%s)
+ // => Don't change the sequence of the above six aggregates!
+ // * (2) group by ([tqs,tqe),IntervalLength) => Make sure (tqe-tqs) is
divisible by
+ // IntervalLength!
+ // * (3) NOTE the time unit of interval. Update for different datasets!
+ private static final String queryFormat =
+ "select min_time(%s), max_time(%s), first_value(%s), last_value(%s),
min_value(%s), max_value(%s) "
+ + "from %s "
+ + "group by ([%d, %d), %d%s)"; // note the time precision unit is
also parameterized
+
+ private static final String queryFormat_UDF =
+ "select M4(%1$s,'tqs'='%3$d','tqe'='%4$d','w'='%5$d') from %2$s where
time>=%3$d and time<%4$d";
+
+ // private static String device = "root.game";
+ // private static String measurement = "s6";
+ // private static TSDataType tsDataType = TSDataType.INT64; //
TSDataType.DOUBLE;
+ // private static String timestamp_precision = "ns"; // ns, us, ms
+ // private static long dataMinTime = 0;
+ // private static long dataMaxTime = 617426057626L;
+ // private static long total_time_length = dataMaxTime - dataMinTime;
+ // private static int total_point_number = 1200000;
+ // private static int iotdb_chunk_point_size = 100000;
+ // private static long chunkAvgTimeLen = (long) Math
+ // .ceil(total_time_length / Math.ceil(total_point_number * 1.0 /
iotdb_chunk_point_size));
+ // private static String filePath =
+ //
+ //
"D:\\github\\m4-lsm\\M4-visualization-exp\\src\\main\\java\\org\\apache\\iotdb\\datasets\\BallSpeed.csv";
+ // private static int deletePercentage = 0; // 0 means no deletes. 0-100
+ // private static int deleteLenPercentage = 0; // 0-100
每次删除的时间长度，用chunkAvgTimeLen的百分比表示
+ // private static int timeIdx = 0; // 时间戳idx，从0开始
+ // private static int valueIdx = 1; // 值idx，从0开始
+ // private static int w = 2;
+ // private static long range = total_time_length;
+ // private static boolean enableRegularityTimeDecode = true;
+ // private static long regularTimeInterval = 511996L;
+ // private static String approach = "mac"; // 选择查询执行算法: 1: MAC, 2: MOC, 3:
CPV
+
+ // private static String device = "root.debs2012";
+ // private static String measurement = "mf03";
+ // private static TSDataType tsDataType = TSDataType.INT64; //
TSDataType.DOUBLE;
+ // private static String timestamp_precision = "ns"; // ns, us, ms
+ // private static long dataMinTime = 1329955200008812200L;
+ // private static long dataMaxTime = 1329965999991045200L;
+ // private static long total_time_length = dataMaxTime - dataMinTime;
+ // private static int total_point_number = 1076102;
+ // private static int iotdb_chunk_point_size = 100000;
+ // private static long chunkAvgTimeLen = (long) Math
+ // .ceil(total_time_length / Math.ceil(total_point_number * 1.0 /
iotdb_chunk_point_size));
+ // private static String filePath =
+ //
+ //
"D:\\github\\m4-lsm\\M4-visualization-exp\\src\\main\\java\\org\\apache\\iotdb\\datasets\\MF03.csv";
+ // private static int deletePercentage = 0; // 0 means no deletes. 0-100
+ // private static int deleteLenPercentage = 0; // 0-100
每次删除的时间长度，用chunkAvgTimeLen的百分比表示
+ // private static int timeIdx = 0; // 时间戳idx，从0开始
+ // private static int valueIdx = 1; // 值idx，从0开始
+ // private static int w = 2;
+ // private static long range = total_time_length;
+ // private static boolean enableRegularityTimeDecode = true;
+ // private static long regularTimeInterval = 10000900L;
+ // private static String approach = "mac"; // 选择查询执行算法: 1: MAC, 2: MOC, 3:
CPV
+
+ private static String device = "root.kobelco.trans.03.1090001603.2401604";
+ private static String measurement = "KOB_0002_00_67";
+ private static TSDataType tsDataType = TSDataType.INT64; //
TSDataType.DOUBLE;
+ private static String timestamp_precision = "ms"; // ns, us, ms
+ private static long dataMinTime = 1616805035973L;
+ private static long dataMaxTime = 1627380839563L;
+ private static long total_time_length = dataMaxTime - dataMinTime;
+ private static int total_point_number = 1943180;
+ private static int iotdb_chunk_point_size = 100000;
+ private static long chunkAvgTimeLen =
+ (long)
+ Math.ceil(
+ total_time_length / Math.ceil(total_point_number * 1.0 /
iotdb_chunk_point_size));
+ private static String filePath =
+
"D:\\github\\m4-lsm\\M4-visualization-exp\\src\\main\\java\\org\\apache\\iotdb\\datasets\\KOB_2.csv";
+ private static int deletePercentage = 0; // 0 means no deletes. 0-100
+ private static int deleteLenPercentage = 0; // 0-100
每次删除的时间长度，用chunkAvgTimeLen的百分比表示
+ private static int timeIdx = 0; // 时间戳idx，从0开始
+ private static int valueIdx = 1; // 值idx，从0开始
+ private static int w = 3;
+ private static long range = total_time_length;
+ private static boolean enableRegularityTimeDecode = true;
+ private static long regularTimeInterval = 1000L;
+ private static String approach = "mac"; // 选择查询执行算法: 1: MAC, 2: MOC, 3: CPV
+
+ @Before
+ public void setUp() throws Exception {
+ // config.setEnableCPV(true);
+ // config.setTimestampPrecision(timestamp_precision);
+ // config.setAvgSeriesPointNumberThreshold(iotdb_chunk_point_size);
+ // config.setUnSeqTsFileSize(1073741824);
+ // config.setSeqTsFileSize(1073741824);
+ // config.setCompactionStrategy(CompactionStrategy.NO_COMPACTION);
+ // config.setEnableUnseqCompaction(false);
+ // config.setEnablePerformanceStat(false);
+ //
+ // TSFileDescriptor.getInstance().getConfig()
+ // .setEnableRegularityTimeDecode(enableRegularityTimeDecode);
+ //
TSFileDescriptor.getInstance().getConfig().setRegularTimeInterval(regularTimeInterval);
+ //
TSFileDescriptor.getInstance().getConfig().setPageSizeInByte(1073741824);
+ //
+ // EnvironmentUtils.envSetUp(); // start after configuration settings
+ // Class.forName(Config.JDBC_DRIVER_NAME);
+ //
+ // System.out.println("[WriteData] device=" + device);
+ // System.out.println("[WriteData] measurement=" + measurement);
+ // System.out.println("[WriteData] dataType=" + tsDataType);
+ // System.out.println("[WriteData] timestamp_precision=" +
timestamp_precision);
+ // System.out.println("[WriteData] dataMinTime=" + dataMinTime);
+ // System.out.println("[WriteData] dataMaxTime=" + dataMaxTime);
+ // System.out.println("[WriteData] total_time_length=" +
total_time_length);
+ // System.out.println("[WriteData] total_point_number=" +
total_point_number);
+ // System.out.println("[WriteData] iotdb_chunk_point_size=" +
iotdb_chunk_point_size);
+ // System.out.println("[WriteData] derived estimated chunkAvgTimeLen ="
+ chunkAvgTimeLen);
+ // System.out.println("[WriteData] filePath=" + filePath);
+ // System.out.println("[WriteData] deletePercentage=" +
deletePercentage);
+ // System.out.println("[WriteData] deleteLenPercentage=" +
deleteLenPercentage);
+ // System.out.println("[WriteData] timeIdx=" + timeIdx);
+ // System.out.println("[WriteData] valueIdx=" + valueIdx);
+ // System.out.println(
+ // "[WriteData] enableRegularityTimeDecode="
+ // +
TSFileDescriptor.getInstance().getConfig().isEnableRegularityTimeDecode());
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ // EnvironmentUtils.cleanEnv();
+ }
+
+ /** Before writing data, make sure check the server parameter
configurations. */
+ // Usage: java -jar WriteData-0.12.4.jar device measurement dataType
timestamp_precision
+ // total_time_length total_point_number iotdb_chunk_point_size filePath
deleteFreq deleteLen
+ // timeIdx valueIdx
+ @Test
+ public void test1() throws Exception {
+ // System.out.println("writing data...");
+ // writeData();
+
+ System.out.println("querying data...");
+ System.out.println("[QueryData] query range=" + range);
+ System.out.println("[QueryData] w=" + w);
+
+ if (!approach.equals("mac") && !approach.equals("moc") &&
!approach.equals("cpv")) {
+ throw new IOException("Approach wrong. Only accepts mac/moc/cpv");
+ }
+ System.out.println("[QueryData] approach=" + approach);
+ if (approach.equals("moc")) {
+ System.out.println(
+ "MAKE SURE you have set the enable_CPV as false in
`iotdb-engine.properties` for MOC!");
+ Assert.assertFalse(config.isEnableCPV());
+ } else if (approach.equals("cpv")) {
+ System.out.println(
+ "MAKE SURE you have set the enable_CPV as true in
`iotdb-engine.properties` for CPV!");
+ Assert.assertTrue(config.isEnableCPV());
+ }
+
+ long minTime;
+ long maxTime;
+ long interval;
+ if (range >= (dataMaxTime - dataMinTime)) {
+ minTime = dataMinTime;
+ interval = (long) Math.ceil((double) (dataMaxTime - dataMinTime) / w);
+ } else {
+ // randomize between [dataMinTime, dataMaxTime-range]
+ minTime =
+ (long) Math.ceil(dataMinTime + Math.random() * (dataMaxTime - range
- dataMinTime + 1));
+ interval = (long) Math.ceil((double) range / w);
+ }
+ maxTime = minTime + interval * w;
+
+ Session session = new Session("127.0.0.1", 6667, "root", "root");
+ session.open(false);
+
+ // Set it big to avoid multiple fetch, which is very important.
+ // Because the IOMonitor implemented in IoTDB does not cover the
fetchResults operator yet.
+ // As M4 already does data reduction, so even the w is very big such as
8000, the returned
+ // query result size is no more than 8000*4=32000.
+ session.setFetchSize(1000000);
+
+ String sql;
+ if (approach.equals("mac")) {
+ // MAC UDF
+ sql = String.format(queryFormat_UDF, measurement, device, minTime,
maxTime, w); // MAC
+ } else {
+ // MOC and CPV sql use the same sql queryFormat.
+ sql =
+ String.format(
+ queryFormat,
+ measurement,
+ measurement,
+ measurement,
+ measurement,
+ measurement,
+ measurement,
+ device,
+ minTime,
+ maxTime,
+ interval,
+ timestamp_precision); // note the time precision unit
+ }
+ System.out.println("[QueryData] sql=" + sql);
+
+ long c = 0;
+ SessionDataSet dataSet = session.executeQueryStatement(sql);
+ DataIterator iterator = dataSet.iterator();
+ // System.out.println(dataSet.getColumnNames());
+ while (iterator.next()) { // this way avoid constructing rowRecord
+ c++;
+ String ans;
+ if (approach.equals("mac")) {
+ ans =
+ String.format(
+ "%s,%s",
+ iterator.getString(1), // time
+ iterator.getString(2)); // M4
+ } else {
+ ans =
+ String.format(
+ "%s,%s,%s,%s,%s,%s,%s",
+ iterator.getString(1), // time
+ iterator.getString(2), // min_time
+ iterator.getString(3), // max_time
+ iterator.getString(4), // first_value
+ iterator.getString(5), // last_value
+ iterator.getString(6), // min_value & bottomTime
+ iterator.getString(7)); // max_value & topTime
+ }
+ System.out.println(ans);
+ }
+ Assert.assertEquals(w, c);
+
+ // session.executeNonQueryStatement("clear cache");
+ dataSet = session.executeFinish();
+ String info = dataSet.getFinishResult();
+ System.out.println(
+ info); // don't add more string to this output, as ProcessResult code
depends on this.
+ System.out.println("[QueryData] query result line number=" + c);
+ dataSet.closeOperationHandle();
+ session.close();
+
+ DecimalFormat df = new DecimalFormat("#,###.00");
+ double max = timeColumnTS2DIFFLoadBatchCost.getMax();
+ double min = timeColumnTS2DIFFLoadBatchCost.getMin();
+ double mean = timeColumnTS2DIFFLoadBatchCost.getMean();
+ double std = timeColumnTS2DIFFLoadBatchCost.getStandardDeviation();
+ double p25 = timeColumnTS2DIFFLoadBatchCost.getPercentile(25);
+ double p50 = timeColumnTS2DIFFLoadBatchCost.getPercentile(50);
+ double p75 = timeColumnTS2DIFFLoadBatchCost.getPercentile(75);
+ double p90 = timeColumnTS2DIFFLoadBatchCost.getPercentile(90);
+ double p95 = timeColumnTS2DIFFLoadBatchCost.getPercentile(95);
+ System.out.println(
+ "timeColumnTS2DIFFLoadBatchCost_stats"
+ + ": "
+ + "num="
+ + timeColumnTS2DIFFLoadBatchCost.getN()
+ + ", "
+ // num is inaccurate because I let alone the last chunk
+ + "sum="
+ + df.format(timeColumnTS2DIFFLoadBatchCost.getSum())
+ + "us,"
+ + "mean="
+ + df.format(mean)
+ + ", "
+ + "min="
+ + df.format(min)
+ + ", "
+ + "max="
+ + df.format(max)
+ + ", "
+ + "std="
+ + df.format(std)
+ + ", "
+ + "p25="
+ + df.format(p25)
+ + ", "
+ + "p50="
+ + df.format(p50)
+ + ", "
+ + "p75="
+ + df.format(p75)
+ + ", "
+ + "p90="
+ + df.format(p90)
+ + ", "
+ + "p95="
+ + df.format(p95));
+
+ System.out.println("Equal Num: " + countForRegularEqual.getN());
+ System.out.println("NOT Equal Num: " + countForRegularNOTEqual.getN());
+
+ System.out.println("hit Num: " + countForHitNewDeltas.getN());
+ System.out.println("NOT hit Num: " + countForNotHitNewDeltas.getN());
+
+ max = countForRegularNewDeltas.getMax();
+ min = countForRegularNewDeltas.getMin();
+ mean = countForRegularNewDeltas.getMean();
+ std = countForRegularNewDeltas.getStandardDeviation();
+ p25 = countForRegularNewDeltas.getPercentile(25);
+ p50 = countForRegularNewDeltas.getPercentile(50);
+ p75 = countForRegularNewDeltas.getPercentile(75);
+ p90 = countForRegularNewDeltas.getPercentile(90);
+ p95 = countForRegularNewDeltas.getPercentile(95);
+ System.out.println(
+ "countForRegularNewDeltas_stats"
+ + ": "
+ + "num="
+ + countForRegularNewDeltas.getN()
+ + ", "
+ // num is inaccurate because I let alone the last chunk
+ + "sum="
+ + df.format(countForRegularNewDeltas.getSum())
+ + "us,"
+ + "mean="
+ + df.format(mean)
+ + ", "
+ + "min="
+ + df.format(min)
+ + ", "
+ + "max="
+ + df.format(max)
+ + ", "
+ + "std="
+ + df.format(std)
+ + ", "
+ + "p25="
+ + df.format(p25)
+ + ", "
+ + "p50="
+ + df.format(p50)
+ + ", "
+ + "p75="
+ + df.format(p75)
+ + ", "
+ + "p90="
+ + df.format(p90)
+ + ", "
+ + "p95="
+ + df.format(p95));
+
+ max = byteArrayLengthStatistics.getMax();
+ min = byteArrayLengthStatistics.getMin();
+ mean = byteArrayLengthStatistics.getMean();
+ std = byteArrayLengthStatistics.getStandardDeviation();
+ p25 = byteArrayLengthStatistics.getPercentile(25);
+ p50 = byteArrayLengthStatistics.getPercentile(50);
+ p75 = byteArrayLengthStatistics.getPercentile(75);
+ p90 = byteArrayLengthStatistics.getPercentile(90);
+ p95 = byteArrayLengthStatistics.getPercentile(95);
+ System.out.println(
+ "byteArrayLengthStatistics_stats"
+ + ": "
+ + "num="
+ + byteArrayLengthStatistics.getN()
+ + ", "
+ // num is inaccurate because I let alone the last chunk
+ + "sum="
+ + df.format(byteArrayLengthStatistics.getSum())
+ + "us,"
+ + "mean="
+ + df.format(mean)
+ + ", "
+ + "min="
+ + df.format(min)
+ + ", "
+ + "max="
+ + df.format(max)
+ + ", "
+ + "std="
+ + df.format(std)
+ + ", "
+ + "p25="
+ + df.format(p25)
+ + ", "
+ + "p50="
+ + df.format(p50)
+ + ", "
+ + "p75="
+ + df.format(p75)
+ + ", "
+ + "p90="
+ + df.format(p90)
+ + ", "
+ + "p95="
+ + df.format(p95));
+
+ System.out.println("allRegularBytes size: " +
allRegularBytesSize.getMax());
+ System.out.println("prepare AllRegularBytes cost: " +
prepareAllRegulars.getSum() + "us");
+ }
+
+ public void writeData()
+ throws IoTDBConnectionException, StatementExecutionException,
IOException {
+
+ if (deletePercentage < 0 || deletePercentage > 100) {
+ throw new IOException("WRONG deletePercentage!");
+ }
+ if (deleteLenPercentage < 0 || deleteLenPercentage > 100) {
+ throw new IOException("WRONG deleteLenPercentage!");
+ }
+
+ int deletePeriod =
+ (int)
+ Math.floor(
+ 100
+ * 1.0
+ / deletePercentage
+ * iotdb_chunk_point_size); // use period to control
percentage
+ long deleteLen = (long) Math.floor(chunkAvgTimeLen * deleteLenPercentage *
1.0 / 100);
+
+ List<String> deletePaths = new ArrayList<>();
+ deletePaths.add(device + "." + measurement);
+
+ Session session = new Session("127.0.0.1", 6667, "root", "root");
+ session.open(false);
+
+ // this is to make all following inserts unseq chunks
+ if (timestamp_precision.toLowerCase().equals("ns")) {
+ session.insertRecord(
+ device,
+ 1644181628000000000L, // ns
+ // NOTE UPDATE TIME DATATYPE! [[update]]. DONT USE System.nanoTime()!
+ Collections.singletonList(measurement),
+ Collections.singletonList(tsDataType), // NOTE UPDATE VALUE DATATYPE!
+ parseValue("0", tsDataType)); // NOTE UPDATE VALUE DATATYPE!
+ } else if (timestamp_precision.toLowerCase().equals("us")) {
+ session.insertRecord(
+ device,
+ 1644181628000000L, // us
+ // NOTE UPDATE TIME DATATYPE! [[update]]. DONT USE System.nanoTime()!
+ Collections.singletonList(measurement),
+ Collections.singletonList(tsDataType), // NOTE UPDATE VALUE DATATYPE!
+ parseValue("0", tsDataType)); // NOTE UPDATE VALUE DATATYPE!
+ } else { // ms
+ session.insertRecord(
+ device,
+ 1644181628000L, // ms
+ // NOTE UPDATE TIME DATATYPE! [[update]]. DONT USE System.nanoTime()!
+ Collections.singletonList(measurement),
+ Collections.singletonList(tsDataType), // NOTE UPDATE VALUE DATATYPE!
+ parseValue("0", tsDataType)); // NOTE UPDATE VALUE DATATYPE!
+ }
+ session.executeNonQueryStatement("flush");
+
+ File f = new File(filePath);
+ String line = null;
+ BufferedReader reader = new BufferedReader(new FileReader(f));
+ long lastDeleteMinTime = Long.MAX_VALUE;
+ long lastDeleteMaxTime = Long.MIN_VALUE;
+ int cnt = 0;
+ while ((line = reader.readLine()) != null) {
+ String[] split = line.split(",");
+ long timestamp = Long.parseLong(split[timeIdx]);
+ session.insertRecord(
+ device,
+ timestamp,
+ Collections.singletonList(measurement),
+ Collections.singletonList(tsDataType),
+ parseValue(split[valueIdx], tsDataType));
+ cnt++;
+
+ if (timestamp > lastDeleteMaxTime) {
+ lastDeleteMaxTime = timestamp;
+ }
+ if (timestamp < lastDeleteMinTime) {
+ lastDeleteMinTime = timestamp;
+ }
+
+ if (deletePercentage != 0) {
+ if (cnt >= deletePeriod) {
+ cnt = 0;
+ // randomize deleteStartTime in [lastMinTime,
max(lastMaxTime-deleteLen,lastMinTime+1)]
+ long rightBound = Math.max(lastDeleteMaxTime - deleteLen,
lastDeleteMinTime + 1);
+ long deleteStartTime =
+ (long)
+ Math.ceil(
+ lastDeleteMinTime + Math.random() * (rightBound -
lastDeleteMinTime + 1));
+ long deleteEndTime = deleteStartTime + deleteLen - 1;
+ session.deleteData(deletePaths, deleteStartTime, deleteEndTime);
+ System.out.println("[[[[delete]]]]]" + deleteStartTime + "," +
deleteEndTime);
+
+ lastDeleteMinTime = Long.MAX_VALUE;
+ lastDeleteMaxTime = Long.MIN_VALUE;
+ }
+ }
+ }
+ session.executeNonQueryStatement("flush");
+ session.close();
+ }
+
+ public static Object parseValue(String value, TSDataType tsDataType) throws
IOException {
+ if (tsDataType == TSDataType.INT64) {
+ return Long.parseLong(value);
+ } else if (tsDataType == TSDataType.DOUBLE) {
+ return Double.parseDouble(value);
+ } else {
+ throw new IOException("data type wrong");
+ }
+ }
+}
diff --git
a/session/src/test/java/org/apache/iotdb/session/MyRealDataWriteQueryTest.java
b/session/src/test/java/org/apache/iotdb/session/MyRealDataWriteQueryTest.java
index f2008592e3..7efce9a4ed 100644
---
a/session/src/test/java/org/apache/iotdb/session/MyRealDataWriteQueryTest.java
+++
b/session/src/test/java/org/apache/iotdb/session/MyRealDataWriteQueryTest.java
@@ -1,12 +1,5 @@
package org.apache.iotdb.session;
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.compaction.CompactionStrategy;
@@ -17,11 +10,31 @@ import org.apache.iotdb.rpc.StatementExecutionException;
import org.apache.iotdb.session.SessionDataSet.DataIterator;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.text.DecimalFormat;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import static
org.apache.iotdb.tsfile.common.constant.TsFileConstant.allRegularBytesSize;
+import static
org.apache.iotdb.tsfile.common.constant.TsFileConstant.byteArrayLengthStatistics;
+import static
org.apache.iotdb.tsfile.common.constant.TsFileConstant.countForHitNewDeltas;
+import static
org.apache.iotdb.tsfile.common.constant.TsFileConstant.countForNotHitNewDeltas;
+import static
org.apache.iotdb.tsfile.common.constant.TsFileConstant.countForRegularEqual;
+import static
org.apache.iotdb.tsfile.common.constant.TsFileConstant.countForRegularNOTEqual;
+import static
org.apache.iotdb.tsfile.common.constant.TsFileConstant.countForRegularNewDeltas;
+import static
org.apache.iotdb.tsfile.common.constant.TsFileConstant.prepareAllRegulars;
+import static
org.apache.iotdb.tsfile.common.constant.TsFileConstant.timeColumnTS2DIFFLoadBatchCost;
+
public class MyRealDataWriteQueryTest {
private static final IoTDBConfig config =
IoTDBDescriptor.getInstance().getConfig();
@@ -39,28 +52,78 @@ public class MyRealDataWriteQueryTest {
private static final String queryFormat_UDF =
"select M4(%1$s,'tqs'='%3$d','tqe'='%4$d','w'='%5$d') from %2$s where
time>=%3$d and time<%4$d";
- private static String device = "root.game";
- private static String measurement = "s6";
+// private static String device = "root.game";
+// private static String measurement = "s6";
+// private static TSDataType tsDataType = TSDataType.INT64; //
TSDataType.DOUBLE;
+// private static String timestamp_precision = "ns"; // ns, us, ms
+// private static long dataMinTime = 0;
+// private static long dataMaxTime = 617426057626L;
+// private static long total_time_length = dataMaxTime - dataMinTime;
+// private static int total_point_number = 1200000;
+// private static int iotdb_chunk_point_size = 100000;
+// private static long chunkAvgTimeLen = (long) Math
+// .ceil(total_time_length / Math.ceil(total_point_number * 1.0 /
iotdb_chunk_point_size));
+// private static String filePath =
+//
+//
"D:\\github\\m4-lsm\\M4-visualization-exp\\src\\main\\java\\org\\apache\\iotdb\\datasets\\BallSpeed.csv";
+// private static int deletePercentage = 0; // 0 means no deletes. 0-100
+// private static int deleteLenPercentage = 0; // 0-100
每次删除的时间长度，用chunkAvgTimeLen的百分比表示
+// private static int timeIdx = 0; // 时间戳idx，从0开始
+// private static int valueIdx = 1; // 值idx，从0开始
+// private static int w = 2;
+// private static long range = total_time_length;
+// private static boolean enableRegularityTimeDecode = false;
+// private static long regularTimeInterval = 511996L;
+// private static String approach = "mac"; // 选择查询执行算法: 1: MAC, 2: MOC, 3:
CPV
+
+ // private static String device = "root.debs2012";
+ // private static String measurement = "mf03";
+ // private static TSDataType tsDataType = TSDataType.INT64; //
TSDataType.DOUBLE;
+ // private static String timestamp_precision = "ns"; // ns, us, ms
+ // private static long dataMinTime = 1329955200008812200L;
+ // private static long dataMaxTime = 1329965999991045200L;
+ // private static long total_time_length = dataMaxTime - dataMinTime;
+ // private static int total_point_number = 1076102;
+ // private static int iotdb_chunk_point_size = 100000;
+ // private static long chunkAvgTimeLen = (long) Math
+ // .ceil(total_time_length / Math.ceil(total_point_number * 1.0 /
iotdb_chunk_point_size));
+ // private static String filePath =
+ //
+ //
"D:\\github\\m4-lsm\\M4-visualization-exp\\src\\main\\java\\org\\apache\\iotdb\\datasets\\MF03.csv";
+ // private static int deletePercentage = 0; // 0 means no deletes. 0-100
+ // private static int deleteLenPercentage = 0; // 0-100
每次删除的时间长度，用chunkAvgTimeLen的百分比表示
+ // private static int timeIdx = 0; // 时间戳idx，从0开始
+ // private static int valueIdx = 1; // 值idx，从0开始
+ // private static int w = 2;
+ // private static long range = total_time_length;
+ // private static boolean enableRegularityTimeDecode = true;
+ // private static long regularTimeInterval = 10000900L;
+ // private static String approach = "mac"; // 选择查询执行算法: 1: MAC, 2: MOC, 3:
CPV
+
+ private static String device = "root.kobelco.trans.03.1090001603.2401604";
+ private static String measurement = "KOB_0002_00_67";
private static TSDataType tsDataType = TSDataType.INT64; //
TSDataType.DOUBLE;
- private static String timestamp_precision = "ns"; // ns, us, ms
- private static long dataMinTime = 0;
- private static long dataMaxTime = 617426057626L;
+ private static String timestamp_precision = "ms"; // ns, us, ms
+ private static long dataMinTime = 1616805035973L;
+ private static long dataMaxTime = 1627380839563L;
private static long total_time_length = dataMaxTime - dataMinTime;
- private static int total_point_number = 1200000;
+ private static int total_point_number = 1943180;
private static int iotdb_chunk_point_size = 100000;
- private static long chunkAvgTimeLen = (long) Math
- .ceil(total_time_length / Math.ceil(total_point_number * 1.0 /
iotdb_chunk_point_size));
+ private static long chunkAvgTimeLen =
+ (long)
+ Math.ceil(
+ total_time_length / Math.ceil(total_point_number * 1.0 /
iotdb_chunk_point_size));
private static String filePath =
-
"D:\\github\\m4-lsm\\M4-visualization-exp\\src\\main\\java\\org\\apache\\iotdb\\datasets\\BallSpeed.csv";
+
"D:\\github\\m4-lsm\\M4-visualization-exp\\src\\main\\java\\org\\apache\\iotdb\\datasets\\KOB_2.csv";
private static int deletePercentage = 0; // 0 means no deletes. 0-100
private static int deleteLenPercentage = 0; // 0-100
每次删除的时间长度，用chunkAvgTimeLen的百分比表示
private static int timeIdx = 0; // 时间戳idx，从0开始
private static int valueIdx = 1; // 值idx，从0开始
- private static int w = 2;
+ private static int w = 3;
private static long range = total_time_length;
- private static boolean enableRegularityTimeDecode = true;
- private static long regularTimeInterval = 511996L;
- private static String approach = "cpv"; // 选择查询执行算法: 1: MAC, 2: MOC, 3: CPV
+ private static boolean enableRegularityTimeDecode = false;
+ private static long regularTimeInterval = 1000L;
+ private static String approach = "mac"; // 选择查询执行算法: 1: MAC, 2: MOC, 3: CPV
@Before
public void setUp() throws Exception {
@@ -73,7 +136,8 @@ public class MyRealDataWriteQueryTest {
config.setEnableUnseqCompaction(false);
config.setEnablePerformanceStat(false);
- TSFileDescriptor.getInstance().getConfig()
+ TSFileDescriptor.getInstance()
+ .getConfig()
.setEnableRegularityTimeDecode(enableRegularityTimeDecode);
TSFileDescriptor.getInstance().getConfig().setRegularTimeInterval(regularTimeInterval);
TSFileDescriptor.getInstance().getConfig().setPageSizeInByte(1073741824);
@@ -103,12 +167,10 @@ public class MyRealDataWriteQueryTest {
@After
public void tearDown() throws Exception {
- EnvironmentUtils.cleanEnv();
+ // EnvironmentUtils.cleanEnv();
}
- /**
- * Before writing data, make sure check the server parameter configurations.
- */
+ /** Before writing data, make sure check the server parameter
configurations. */
// Usage: java -jar WriteData-0.12.4.jar device measurement dataType
timestamp_precision
// total_time_length total_point_number iotdb_chunk_point_size filePath
deleteFreq deleteLen
// timeIdx valueIdx
@@ -218,6 +280,154 @@ public class MyRealDataWriteQueryTest {
System.out.println("[QueryData] query result line number=" + c);
dataSet.closeOperationHandle();
session.close();
+
+ DecimalFormat df = new DecimalFormat("#,###.00");
+ double max = timeColumnTS2DIFFLoadBatchCost.getMax();
+ double min = timeColumnTS2DIFFLoadBatchCost.getMin();
+ double mean = timeColumnTS2DIFFLoadBatchCost.getMean();
+ double std = timeColumnTS2DIFFLoadBatchCost.getStandardDeviation();
+ double p25 = timeColumnTS2DIFFLoadBatchCost.getPercentile(25);
+ double p50 = timeColumnTS2DIFFLoadBatchCost.getPercentile(50);
+ double p75 = timeColumnTS2DIFFLoadBatchCost.getPercentile(75);
+ double p90 = timeColumnTS2DIFFLoadBatchCost.getPercentile(90);
+ double p95 = timeColumnTS2DIFFLoadBatchCost.getPercentile(95);
+ System.out.println(
+ "timeColumnTS2DIFFLoadBatchCost_stats"
+ + ": "
+ + "num="
+ + timeColumnTS2DIFFLoadBatchCost.getN()
+ + ", "
+ // num is inaccurate because I let alone the last chunk
+ + "sum="
+ + df.format(timeColumnTS2DIFFLoadBatchCost.getSum())
+ + "us,"
+ + "mean="
+ + df.format(mean)
+ + ", "
+ + "min="
+ + df.format(min)
+ + ", "
+ + "max="
+ + df.format(max)
+ + ", "
+ + "std="
+ + df.format(std)
+ + ", "
+ + "p25="
+ + df.format(p25)
+ + ", "
+ + "p50="
+ + df.format(p50)
+ + ", "
+ + "p75="
+ + df.format(p75)
+ + ", "
+ + "p90="
+ + df.format(p90)
+ + ", "
+ + "p95="
+ + df.format(p95));
+
+ System.out.println("Equal Num: " + countForRegularEqual.getN());
+ System.out.println("NOT Equal Num: " + countForRegularNOTEqual.getN());
+
+ System.out.println("hit Num: " + countForHitNewDeltas.getN());
+ System.out.println("NOT hit Num: " + countForNotHitNewDeltas.getN());
+
+ max = countForRegularNewDeltas.getMax();
+ min = countForRegularNewDeltas.getMin();
+ mean = countForRegularNewDeltas.getMean();
+ std = countForRegularNewDeltas.getStandardDeviation();
+ p25 = countForRegularNewDeltas.getPercentile(25);
+ p50 = countForRegularNewDeltas.getPercentile(50);
+ p75 = countForRegularNewDeltas.getPercentile(75);
+ p90 = countForRegularNewDeltas.getPercentile(90);
+ p95 = countForRegularNewDeltas.getPercentile(95);
+ System.out.println(
+ "countForRegularNewDeltas_stats"
+ + ": "
+ + "num="
+ + countForRegularNewDeltas.getN()
+ + ", "
+ // num is inaccurate because I let alone the last chunk
+ + "sum="
+ + df.format(countForRegularNewDeltas.getSum())
+ + "us,"
+ + "mean="
+ + df.format(mean)
+ + ", "
+ + "min="
+ + df.format(min)
+ + ", "
+ + "max="
+ + df.format(max)
+ + ", "
+ + "std="
+ + df.format(std)
+ + ", "
+ + "p25="
+ + df.format(p25)
+ + ", "
+ + "p50="
+ + df.format(p50)
+ + ", "
+ + "p75="
+ + df.format(p75)
+ + ", "
+ + "p90="
+ + df.format(p90)
+ + ", "
+ + "p95="
+ + df.format(p95));
+
+ max = byteArrayLengthStatistics.getMax();
+ min = byteArrayLengthStatistics.getMin();
+ mean = byteArrayLengthStatistics.getMean();
+ std = byteArrayLengthStatistics.getStandardDeviation();
+ p25 = byteArrayLengthStatistics.getPercentile(25);
+ p50 = byteArrayLengthStatistics.getPercentile(50);
+ p75 = byteArrayLengthStatistics.getPercentile(75);
+ p90 = byteArrayLengthStatistics.getPercentile(90);
+ p95 = byteArrayLengthStatistics.getPercentile(95);
+ System.out.println(
+ "byteArrayLengthStatistics_stats"
+ + ": "
+ + "num="
+ + byteArrayLengthStatistics.getN()
+ + ", "
+ // num is inaccurate because I let alone the last chunk
+ + "sum="
+ + df.format(byteArrayLengthStatistics.getSum())
+ + "us,"
+ + "mean="
+ + df.format(mean)
+ + ", "
+ + "min="
+ + df.format(min)
+ + ", "
+ + "max="
+ + df.format(max)
+ + ", "
+ + "std="
+ + df.format(std)
+ + ", "
+ + "p25="
+ + df.format(p25)
+ + ", "
+ + "p50="
+ + df.format(p50)
+ + ", "
+ + "p75="
+ + df.format(p75)
+ + ", "
+ + "p90="
+ + df.format(p90)
+ + ", "
+ + "p95="
+ + df.format(p95));
+
+ System.out.println("allRegularBytes size: " +
allRegularBytesSize.getMax());
+ System.out.println("prepare AllRegularBytes cost: " +
prepareAllRegulars.getSum() + "us");
}
public void writeData()
diff --git
a/session/src/test/java/org/apache/iotdb/session/MySmallRealDataWriteQueryTest.java
b/session/src/test/java/org/apache/iotdb/session/MySmallRealDataWriteQueryTest.java
index 195aff618f..ea35d5e91f 100644
---
a/session/src/test/java/org/apache/iotdb/session/MySmallRealDataWriteQueryTest.java
+++
b/session/src/test/java/org/apache/iotdb/session/MySmallRealDataWriteQueryTest.java
@@ -1,12 +1,5 @@
package org.apache.iotdb.session;
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.compaction.CompactionStrategy;
@@ -17,11 +10,20 @@ import org.apache.iotdb.rpc.StatementExecutionException;
import org.apache.iotdb.session.SessionDataSet.DataIterator;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
public class MySmallRealDataWriteQueryTest {
private static final IoTDBConfig config =
IoTDBDescriptor.getInstance().getConfig();
@@ -48,8 +50,10 @@ public class MySmallRealDataWriteQueryTest {
private static long total_time_length = dataMaxTime - dataMinTime;
private static int total_point_number = 50000;
private static int iotdb_chunk_point_size = 10000;
- private static long chunkAvgTimeLen = (long) Math
- .ceil(total_time_length / Math.ceil(total_point_number * 1.0 /
iotdb_chunk_point_size));
+ private static long chunkAvgTimeLen =
+ (long)
+ Math.ceil(
+ total_time_length / Math.ceil(total_point_number * 1.0 /
iotdb_chunk_point_size));
private static String filePath =
"D:\\github\\m4-lsm\\iotdb\\session\\src\\test\\java\\org\\apache\\iotdb\\session\\BallSpeedSmallData.csv";
private static int deletePercentage = 0; // 0 means no deletes. 0-100
@@ -60,7 +64,7 @@ public class MySmallRealDataWriteQueryTest {
private static long range = total_time_length;
private static boolean enableRegularityTimeDecode = false;
private static long regularTimeInterval = 511996L;
-// private static long regularTimeInterval = 511997L;
+ // private static long regularTimeInterval = 511997L;
private static String approach = "cpv"; // 选择查询执行算法: 1: MAC, 2: MOC, 3: CPV
@Before
@@ -74,7 +78,8 @@ public class MySmallRealDataWriteQueryTest {
config.setEnableUnseqCompaction(false);
config.setEnablePerformanceStat(false);
- TSFileDescriptor.getInstance().getConfig()
+ TSFileDescriptor.getInstance()
+ .getConfig()
.setEnableRegularityTimeDecode(enableRegularityTimeDecode);
TSFileDescriptor.getInstance().getConfig().setRegularTimeInterval(regularTimeInterval);
TSFileDescriptor.getInstance().getConfig().setPageSizeInByte(1073741824);
@@ -107,9 +112,7 @@ public class MySmallRealDataWriteQueryTest {
EnvironmentUtils.cleanEnv();
}
- /**
- * Before writing data, make sure check the server parameter configurations.
- */
+ /** Before writing data, make sure check the server parameter
configurations. */
// Usage: java -jar WriteData-0.12.4.jar device measurement dataType
timestamp_precision
// total_time_length total_point_number iotdb_chunk_point_size filePath
deleteFreq deleteLen
// timeIdx valueIdx
diff --git a/tsfile/pom.xml b/tsfile/pom.xml
index f26c930f1a..a310fc8d38 100644
--- a/tsfile/pom.xml
+++ b/tsfile/pom.xml
@@ -37,6 +37,11 @@
<tsfile.ut.skip>${tsfile.test.skip}</tsfile.ut.skip>
</properties>
<dependencies>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-math3</artifactId>
+ <version>3.6.1</version>
+ </dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/constant/TsFileConstant.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/constant/TsFileConstant.java
index 7b8e024f0d..0425f9d793 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/constant/TsFileConstant.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/constant/TsFileConstant.java
@@ -18,6 +18,8 @@
*/
package org.apache.iotdb.tsfile.common.constant;
+import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics;
+
public class TsFileConstant {
public static final String TSFILE_SUFFIX = ".tsfile";
@@ -30,5 +32,22 @@ public class TsFileConstant {
public static final String PATH_SEPARATER_NO_REGEX = "\\.";
public static final char DOUBLE_QUOTE = '"';
+ public static DescriptiveStatistics timeColumnTS2DIFFLoadBatchCost = new
DescriptiveStatistics();
+
+ public static DescriptiveStatistics countForRegularEqual = new
DescriptiveStatistics();
+ public static DescriptiveStatistics countForRegularNOTEqual = new
DescriptiveStatistics();
+ public static DescriptiveStatistics countForRegularZero = new
DescriptiveStatistics();
+
+ public static DescriptiveStatistics countForRegularNewDeltas = new
DescriptiveStatistics();
+
+ public static DescriptiveStatistics countForHitNewDeltas = new
DescriptiveStatistics();
+ public static DescriptiveStatistics countForNotHitNewDeltas = new
DescriptiveStatistics();
+
+ public static DescriptiveStatistics byteArrayLengthStatistics = new
DescriptiveStatistics();
+
+ public static DescriptiveStatistics allRegularBytesSize = new
DescriptiveStatistics();
+
+ public static DescriptiveStatistics prepareAllRegulars = new
DescriptiveStatistics();
+
private TsFileConstant() {}
}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java
index 53e6103f19..8b09da3bc7 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java
@@ -24,9 +24,11 @@ import java.nio.ByteBuffer;
import java.util.HashMap;
import java.util.Map;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
+import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
import org.apache.iotdb.tsfile.encoding.encoder.DeltaBinaryEncoder;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.utils.BytesUtils;
+import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
/**
@@ -183,8 +185,11 @@ public abstract class DeltaBinaryDecoder extends Decoder {
private boolean enableRegularityTimeDecode;
private long regularTimeInterval;
- private byte[]
- encodedRegularTimeInterval; // it depends on minDeltaBase and bitWidth
of each pack
+ // private byte[]
+ // encodedRegularTimeInterval; // it depends on minDeltaBase and
bitWidth of each pack
+
+ private Map<Pair<Long, Integer>, Map<Integer, byte[]>> allRegularBytes =
+ new HashMap<>(); // <newRegularDelta,packWidth> -> (relativePos->bytes)
public LongDeltaDecoder() {
super();
@@ -214,6 +219,8 @@ public abstract class DeltaBinaryDecoder extends Decoder {
* @return long value
*/
protected long loadIntBatch(ByteBuffer buffer) {
+ long start = System.nanoTime();
+
packNum = ReadWriteIOUtils.readInt(buffer);
packWidth = ReadWriteIOUtils.readInt(buffer);
count++;
@@ -229,81 +236,144 @@ public abstract class DeltaBinaryDecoder extends Decoder
{
allocateDataArray();
if (enableRegularityTimeDecode) {
- // preprocess the regular time interval
long newRegularDelta = regularTimeInterval - minDeltaBase;
- Map<Integer, byte[]> regularBytes = new HashMap<>();
- for (int i = 0; i < 8; i++) {
- // i is the starting position in the byte from high to low bits
- int endPos = i + packWidth - 1; // starting from 0
- int byteNum = endPos / 8 + 1;
- byte[] byteArray = new byte[byteNum];
- // put bit-packed newRegularDelta starting at position i,
- // and pad the front and back with newRegularDeltas
-
- // 1. deal with padding the first byte
- for (int x = i - 1; x >= 0; x--) {
- // y is the position in the bit-packed newRegularDelta,
0->packWidth-1 from low to high bits
- int y = (i - x - 1) % packWidth;
- // get the bit indicated by y pos
- int value = BytesUtils.getLongN(newRegularDelta, y);
- // put the bit indicated by y pos into regularBytes
- // setByte pos is from high to low starting from 0, corresponding
to x
- byteArray[0] = BytesUtils.setByteN(byteArray[0], x, value);
- }
+ // System.out.println("newRegularDelta = " + newRegularDelta);
+ // countForRegularNewDeltas.addValue(newRegularDelta);
- // 2. deal with putting newRegularDeltas
- BytesUtils.longToBytes(newRegularDelta, byteArray, i, packWidth);
-
- // 3. deal with padding the last byte
- for (int x = endPos + 1; x < byteNum * 8; x++) {
- // y is the position in the bit-packed newRegularDelta,
0->packWidth-1 from low to high bits
- int y = packWidth - 1 - (x - endPos - 1) % packWidth;
- // get the bit indicated by y pos
- int value = BytesUtils.getLongN(newRegularDelta, y);
- // put the bit indicated by y pos into regularBytes
- // setByte pos is from high to low starting from 0, corresponding
to x
- byteArray[byteNum - 1] = BytesUtils.setByteN(byteArray[byteNum -
1], x, value);
+ if (packWidth == 0) {
+ for (int i = 0; i < packNum; i++) {
+ data[i] = previous + minDeltaBase; // v=0
+ previous = data[i];
+ // System.out.println("[RL]0");
+ // TsFileConstant.countForRegularZero.addValue(1);
}
+ } else if (newRegularDelta < 0
+ || newRegularDelta
+ >= Math.pow(2, packWidth)) { // no need to compare equality cause
impossible
+ for (int i = 0; i < packNum; i++) {
+ long v = BytesUtils.bytesToLong(deltaBuf, packWidth * i,
packWidth);
+ data[i] = previous + minDeltaBase + v;
+ previous = data[i];
+ // System.out.println("[RL]no");
+ // TsFileConstant.countForRegularNOTEqual.addValue(1);
+ }
+ } else {
+ // long start1 = System.nanoTime();
+ Map<Integer, byte[]> regularBytes;
+ if (allRegularBytes.containsKey(new Pair<>(newRegularDelta,
packWidth))) {
+ regularBytes = allRegularBytes.get(new Pair<>(newRegularDelta,
packWidth));
+ // countForHitNewDeltas.addValue(1);
+ // System.out.println("here");
+ } else {
+ // countForNotHitNewDeltas.addValue(1);
+ // System.out.println("here");
+
+ regularBytes = new HashMap<>();
+ for (int i = 0; i < 8; i++) {
+ // i is the starting position in the byte from high to low bits
+ int endPos = i + packWidth - 1; // starting from 0
+ int byteNum = endPos / 8 + 1;
+ byte[] byteArray = new byte[byteNum];
+ if (newRegularDelta != 0) {
+ // otherwise newRegularDelta=0 so leave byteArray as initial
zeros
+
+ // put bit-packed newRegularDelta starting at position i,
+ // and pad the front and back with newRegularDeltas
+ // 1. deal with padding the first byte
+ for (int x = i - 1; x >= 0; x--) {
+ // y is the position in the bit-packed newRegularDelta,
0->packWidth-1 from low to
+ // high bits
+ int y = (i - x - 1) % packWidth;
+ // get the bit indicated by y pos
+ int value = BytesUtils.getLongN(newRegularDelta, y);
+ // put the bit indicated by y pos into regularBytes
+ // setByte pos is from high to low starting from 0,
corresponding to x
+ byteArray[0] = BytesUtils.setByteN(byteArray[0], x, value);
+ }
+
+ // 2. deal with putting newRegularDeltas
+ BytesUtils.longToBytes(newRegularDelta, byteArray, i,
packWidth);
+
+ // 3. deal with padding the last byte
+ for (int x = endPos + 1; x < byteNum * 8; x++) {
+ // y is the position in the bit-packed newRegularDelta,
0->packWidth-1 from low to
+ // high bits
+ int y = packWidth - 1 - (x - endPos - 1) % packWidth;
+ // get the bit indicated by y pos
+ int value = BytesUtils.getLongN(newRegularDelta, y);
+ // put the bit indicated by y pos into regularBytes
+ // setByte pos is from high to low starting from 0,
corresponding to x
+ byteArray[byteNum - 1] =
BytesUtils.setByteN(byteArray[byteNum - 1], x, value);
+ }
+ }
+ regularBytes.put(i, byteArray);
+ }
+ allRegularBytes.put(new Pair<>(newRegularDelta, packWidth),
regularBytes);
+ }
+ // long elapsedTime1 = System.nanoTime() - start1;
+ // TsFileConstant.prepareAllRegulars.addValue(elapsedTime1
/ 1000.0); // us
- regularBytes.put(i, byteArray);
- }
-
- for (int i = 0; i < packNum; i++) {
- // (1) extract bits from deltaBuf,
- // (2) compare bits with encodedRegularTimeInterval,
- // (3) equal to reuse, else to convert
+ for (int i = 0; i < packNum; i++) {
+ // (1) extract bits from deltaBuf,
+ // (2) compare bits with encodedRegularTimeInterval,
+ // (3) equal to reuse, else to convert
- if (packWidth == 0) {
- data[i] = previous + minDeltaBase; // v=0
- } else {
boolean equal = true;
- int posByteIdx = i * packWidth / 8;
- int pos = i * packWidth % 8; // the starting position in the byte
from high to low bits
+
+ // the starting position in the byte from high to low bits
+ int pos = i * packWidth % 8;
+
byte[] byteArray = regularBytes.get(pos);
+
+ int posByteIdx = i * packWidth / 8;
+ // System.out.println("byteArray length=" +
byteArray.length);
+ // byteArrayLengthStatistics.addValue(byteArray.length);
for (int k = 0; k < byteArray.length; k++, posByteIdx++) {
byte regular = byteArray[k];
byte data = deltaBuf[posByteIdx];
if (regular != data) {
equal = false;
+ // System.out.println("k=" + k);
break;
}
}
+ // int posByteIdx = ((i + 1) * packWidth - 1) / 8;
+ // for (int k = byteArray.length - 1; k >= 0; k--,
posByteIdx--) {
+ // // compare the lower bytes first
+ // byte regular = byteArray[k];
+ // byte data = deltaBuf[posByteIdx];
+ // if (regular != data) {
+ // equal = false;
+ //// System.out.println("k'=" + (byteArray.length -
1 - k));
+ // break;
+ // }
+ // }
+
if (equal) {
data[i] = previous + regularTimeInterval;
-// System.out.println("[RL]equals");
+ // System.out.println("[RL]equals");
+ // TsFileConstant.countForRegularEqual.addValue(1);
} else {
long v = BytesUtils.bytesToLong(deltaBuf, packWidth * i,
packWidth);
data[i] = previous + minDeltaBase + v;
-// System.out.println("[RL]no");
+ // System.out.println("[RL]no");
+ //
TsFileConstant.countForRegularNOTEqual.addValue(1);
}
+// data[i] = previous + regularTimeInterval;
+ previous = data[i];
}
- previous = data[i];
}
} else { // without regularity-aware decoding
readPack();
}
+ long runTime = System.nanoTime() - start; // ns
+ TsFileConstant.timeColumnTS2DIFFLoadBatchCost.addValue(runTime /
1000.0); // us
+
+ // allRegularBytesSize.addValue(allRegularBytes.size());
+ // System.out.println("allRegularBytes size=" +
allRegularBytes.size());
+
return firstValue;
}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DeltaBinaryEncoder.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DeltaBinaryEncoder.java
index abc4221649..f2d60704c6 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DeltaBinaryEncoder.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DeltaBinaryEncoder.java
@@ -19,7 +19,6 @@
package org.apache.iotdb.tsfile.encoding.encoder;
-import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.utils.BytesUtils;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
@@ -103,10 +102,10 @@ public abstract class DeltaBinaryEncoder extends Encoder {
}
writeWidth = calculateBitWidthsForDeltaBlockBuffer();
-// // TODO ceil writeWidth to be a multiple of 8
-// if
(TSFileDescriptor.getInstance().getConfig().isEnableRegularityTimeDecode()) {
-// writeWidth = 8 * (int) Math.ceil((double) (writeWidth) / 8.0);
-// }
+ // // TODO ceil writeWidth to be a multiple of 8
+ // if
(TSFileDescriptor.getInstance().getConfig().isEnableRegularityTimeDecode()) {
+ // writeWidth = 8 * (int) Math.ceil((double) (writeWidth) / 8.0);
+ // }
writeHeaderToBytes();
writeDataWithMinWidth();
