This is an automated email from the ASF dual-hosted git repository. leirui pushed a commit to branch research/LTS-visualization in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 85ce09091447c29409372a37d7d9fcceb96a4952 Author: Lei Rui <[email protected]> AuthorDate: Mon Jan 29 18:25:18 2024 +0800 unify sql --- .../resources/conf/iotdb-engine.properties | 12 ++++---- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 2 +- .../groupby/GroupByWithoutValueFilterDataSet.java | 23 ++++++++++++--- .../iotdb/db/integration/tri/MyTest_ILTS.java | 4 +++ .../iotdb/db/integration/tri/MyTest_LTTB.java | 2 ++ .../apache/iotdb/db/integration/tri/MyTest_M4.java | 34 +++++++++++++--------- .../iotdb/db/integration/tri/MyTest_MinMax.java | 27 ++++++++--------- .../db/integration/tri/MyTest_MinMaxLTTB.java | 2 ++ 8 files changed, 70 insertions(+), 36 deletions(-) diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties index 80b2c384466..ddc789dba73 100644 --- a/server/src/assembly/resources/conf/iotdb-engine.properties +++ b/server/src/assembly/resources/conf/iotdb-engine.properties @@ -22,19 +22,21 @@ # MinMax, MinMaxLTTB, M4, LTTB, ILTS enable_Tri=MinMax +#for p1t=0 p1v=0 pnt=100 pnv=100 -rps=4 +#for MinMaxLTTB +rps=2 -numIterations=8 - -acc_avg=true +#for ILTS +numIterations=4 +acc_avg=false acc_rectangle=true -acc_convex=true +acc_convex=false #################### ### enable CPV diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 2ffc96bf2d0..5ef77660ae4 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -440,7 +440,7 @@ public class IoTDBConfig { private long pnt; private double pnv; - private int rps; + private int rps = 2; public int getNumIterations() { return numIterations; diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java index c34fd6af987..d2a44f5ce0c 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java +++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java @@ -224,7 +224,9 @@ public class GroupByWithoutValueFilterDataSet extends GroupByEngineDataSet { List<AggregateResult> aggregations = executor.calcResult( localCurStartTime, localCurStartTime + interval, startTime, endTime, interval); + int c = 0; for (AggregateResult aggregation : aggregations) { + // ATTENTION only take the first two aggregation fields, which are BPv[BPt], TPv[TPt] // Each row correspond to (bucketLeftBound, minV[bottomT], maxV[topT]) of a MinMax bucket MinMaxInfo minMaxInfo = (MinMaxInfo) aggregation.getResult(); if (minMaxInfo == null) { @@ -234,6 +236,11 @@ public class GroupByWithoutValueFilterDataSet extends GroupByEngineDataSet { times.add(minMaxInfo.timestamp); values.add((Double) minMaxInfo.val); } + c++; + if (c >= 2) { + // ATTENTION only take the first two aggregation fields, which are BPv[BPt], TPv[TPt] + break; + } } } @@ -378,6 +385,8 @@ public class GroupByWithoutValueFilterDataSet extends GroupByEngineDataSet { // concat results into a string record = new RowRecord(0); StringBuilder series = new StringBuilder(); + // 全局首点(对于M4来说全局首尾点只是输出不会影响到其它桶的采点) + series.append(CONFIG.getP1v()).append("[").append(CONFIG.getP1t()).append("]").append(","); for (long localCurStartTime = startTime; localCurStartTime + interval <= endTime; @@ -417,6 +426,9 @@ public class GroupByWithoutValueFilterDataSet extends GroupByEngineDataSet { .append(","); } + // 全局尾点(对于M4来说全局首尾点只是输出不会影响到其它桶的采点) + series.append(CONFIG.getPnv()).append("[").append(CONFIG.getPnt()).append("]").append(","); + // MIN_MAX_INT64 this type for field.setBinaryV(new Binary(value.toString())) // 注意sql第一项一定要是min_value因为以后会用到record.addField(series, TSDataType.MIN_MAX_INT64) // 把所有序列组装成string放在第一行第二列里，否则field类型和TSDataType.MIN_MAX_INT64对不上的会有问题。 @@ -447,6 +459,8 @@ public class GroupByWithoutValueFilterDataSet extends GroupByEngineDataSet { // concat results into a string record = new RowRecord(0); StringBuilder series = new StringBuilder(); + // 全局首点(对于MinMax来说全局首尾点只是输出不会影响到其它桶的采点) + series.append(CONFIG.getP1v()).append("[").append(CONFIG.getP1t()).append("]").append(","); for (long localCurStartTime = startTime; localCurStartTime + interval <= endTime; @@ -464,12 +478,13 @@ public class GroupByWithoutValueFilterDataSet extends GroupByEngineDataSet { startTime, endTime, interval); // attention - for (AggregateResult aggregation : aggregations) { - // Each row correspond to (bucketLeftBound, minV[bottomT], maxV[topT]) of a MinMax bucket - series.append(aggregation.getResult()).append(","); - } + series.append(aggregations.get(0).getResult()).append(","); // BPv[BPt] + series.append(aggregations.get(1).getResult()).append(","); // TPv[TPt] } + // 全局尾点(对于MinMax来说全局首尾点只是输出不会影响到其它桶的采点) + series.append(CONFIG.getPnv()).append("[").append(CONFIG.getPnt()).append("]").append(","); + // MIN_MAX_INT64 this type for field.setBinaryV(new Binary(value.toString())) // 注意sql第一项一定要是min_value因为以后会用到record.addField(series, TSDataType.MIN_MAX_INT64) // 把所有序列组装成string放在第一行第二列里，否则field类型和TSDataType.MIN_MAX_INT64对不上的会有问题。 diff --git a/server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_ILTS.java b/server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_ILTS.java index 87bba5c8603..59bf45b5c82 100644 --- a/server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_ILTS.java +++ b/server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_ILTS.java @@ -107,6 +107,7 @@ public class MyTest_ILTS { statement.execute( "SELECT min_value(s0)" // TODO not real min_value here, actually controlled by enableTri + + ",max_value(s0),min_time(s0), max_time(s0), first_value(s0), last_value(s0)" + " FROM root.vehicle.d0 group by ([2,102),20ms)"); // (102-2)/(7-2)=20ms // note keep no empty buckets @@ -138,6 +139,7 @@ public class MyTest_ILTS { statement.execute( "SELECT min_value(s0)" // TODO not real min_value here, actually controlled by enableTri + + ",max_value(s0),min_time(s0), max_time(s0), first_value(s0), last_value(s0)" + " FROM root.vehicle.d0 group by ([2,102),20ms)"); // (102-2)/(7-2)=20ms // note keep no empty buckets @@ -204,6 +206,7 @@ public class MyTest_ILTS { statement.execute( "SELECT min_value(s0)" // TODO not real min_value here, actually controlled by enableTri + + ",max_value(s0),min_time(s0), max_time(s0), first_value(s0), last_value(s0)" + " FROM root.vehicle.d0 group by ([100,2100),200ms)"); // (tn-t2)/(nout-2)=(2100-100)/(12-2)=2000/10=200 @@ -238,6 +241,7 @@ public class MyTest_ILTS { statement.execute( "SELECT min_value(s0)" // TODO not real min_value here, actually controlled by enableTri + + ",max_value(s0),min_time(s0), max_time(s0), first_value(s0), last_value(s0)" + " FROM root.vehicle.d0 group by ([100,2100),200ms)"); // (tn-t2)/(nout-2)=(2100-100)/(12-2)=2000/10=200 diff --git a/server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_LTTB.java b/server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_LTTB.java index 0354dcb57cf..542a9037c27 100644 --- a/server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_LTTB.java +++ b/server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_LTTB.java @@ -107,6 +107,7 @@ public class MyTest_LTTB { statement.execute( "SELECT min_value(s0)" // TODO not real min_value here, actually controlled by enableTri + + ",max_value(s0),min_time(s0), max_time(s0), first_value(s0), last_value(s0)" + " FROM root.vehicle.d0 group by ([2,106),26ms)"); Assert.assertTrue(hasResultSet); @@ -170,6 +171,7 @@ public class MyTest_LTTB { statement.execute( "SELECT min_value(s0)" // TODO not real min_value here, actually controlled by enableTri + + ",max_value(s0),min_time(s0), max_time(s0), first_value(s0), last_value(s0)" + " FROM root.vehicle.d0 group by ([100,2100),250ms)"); // (tn-t2)/(nout-2)=(2100-100)/(10-2)=2000/8=250 diff --git a/server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_M4.java b/server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_M4.java index 4cc39f797d3..0c30926deaa 100644 --- a/server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_M4.java +++ b/server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_M4.java @@ -24,6 +24,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.compaction.CompactionStrategy; import org.apache.iotdb.db.utils.EnvironmentUtils; import org.apache.iotdb.jdbc.Config; +import org.apache.iotdb.jdbc.IoTDBStatement; import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; import org.junit.After; @@ -77,6 +78,11 @@ public class MyTest_M4 { config.setCompactionStrategy(CompactionStrategy.NO_COMPACTION); config.setEnableTri("M4"); + // 对于M4来说全局首尾点只是输出不会影响到其它桶的采点 + config.setP1t(0); + config.setP1v(0); + config.setPnt(200); + config.setPnv(200); // 但是如果走的是unpackOneChunkMetaData(firstChunkMetadata)就没问题， // 因为它直接用chunk元数据去构造pageReader， @@ -102,13 +108,15 @@ public class MyTest_M4 { public void test1() throws Exception { prepareData1(); String res = - "1.0[20],15.0[2],5.0[1],1.0[20],8.0[25],8.0[25],8.0[25],8.0[25],3.0[54],3.0[54],3.0[54],3.0[54],null,null,null[null],null[null],"; + "0.0[0],1.0[20],15.0[2],5.0[1],1.0[20],8.0[25],8.0[25],8.0[25],8.0[25],3.0[54]," + + "3.0[54],3.0[54],3.0[54],null,null,null[null],null[null],200.0[200],"; try (Connection connection = DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); Statement statement = connection.createStatement()) { boolean hasResultSet = statement.execute( "SELECT min_value(s0), max_value(s0),min_time(s0), max_time(s0), first_value(s0), last_value(s0)" + // do not change sequence + " FROM root.vehicle.d0 group by ([0,100),25ms)"); // 注意需要第一项是min_value因为以后会用到record.addField(series, // TSDataType.MIN_MAX_INT64)把所有序列组装成string放在第一行第二列里 @@ -161,25 +169,24 @@ public class MyTest_M4 { prepareData3(); String res = - "1.0[10],10.0[2],5.0[1],4.0[22],2.0[40],8.0[30],8.0[30],2.0[40],4.0[72],20.0[62],5.0[55],4.0[72],1.0[90],11.0[80],11.0[80],1.0[90],"; + "0.0[0],1.0[10],10.0[2],5.0[1],4.0[22],2.0[40],8.0[30],8.0[30],2.0[40]," + + "4.0[72],20.0[62],5.0[55],4.0[72],1.0[90],11.0[80],11.0[80],1.0[90],200.0[200],"; try (Connection connection = DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); Statement statement = connection.createStatement()) { - boolean hasResultSet = - statement.execute( + ResultSet resultSet = + statement.executeQuery( "SELECT min_value(s0), max_value(s0),min_time(s0), max_time(s0), first_value(s0), last_value(s0)" + " FROM root.vehicle.d0 group by ([0,100),25ms)"); // don't change the // sequence!!! - Assert.assertTrue(hasResultSet); - try (ResultSet resultSet = statement.getResultSet()) { - int i = 0; - while (resultSet.next()) { - String ans = resultSet.getString(2); - System.out.println(ans); - Assert.assertEquals(res, ans); - } + int i = 0; + while (resultSet.next()) { + String ans = resultSet.getString(2); + System.out.println(ans); + Assert.assertEquals(res, ans); } + System.out.println(((IoTDBStatement) statement).executeFinish()); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); @@ -231,7 +238,8 @@ public class MyTest_M4 { prepareData3_2(); String res = - "1.0[10],10.0[2],5.0[1],5.0[20],null,null,null[null],null[null],4.0[72],20.0[62],15.0[60],4.0[72],1.0[90],11.0[80],11.0[80],1.0[90],"; + "0.0[0],1.0[10],10.0[2],5.0[1],5.0[20],null,null,null[null],null[null],4.0[72]," + + "20.0[62],15.0[60],4.0[72],1.0[90],11.0[80],11.0[80],1.0[90],200.0[200],"; try (Connection connection = DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); Statement statement = connection.createStatement()) { diff --git a/server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_MinMax.java b/server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_MinMax.java index 06b0432142d..a6f76b440ed 100644 --- a/server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_MinMax.java +++ b/server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_MinMax.java @@ -74,6 +74,11 @@ public class MyTest_MinMax { config.setCompactionStrategy(CompactionStrategy.NO_COMPACTION); config.setEnableTri("MinMax"); + // 对于MinMax来说全局首尾点只是输出不会影响到其它桶的采点 + config.setP1t(0); + config.setP1v(0); + config.setPnt(200); + config.setPnv(200); // 但是如果走的是unpackOneChunkMetaData(firstChunkMetadata)就没问题， // 因为它直接用chunk元数据去构造pageReader， @@ -98,16 +103,15 @@ public class MyTest_MinMax { @Test public void test1() throws Exception { prepareData1(); - // String[] res = new String[]{"0,1[20],15[2]", "25,8[25],8[25]", "50,3[54],3[54]", - // "75,null,null"}; - String res = "1.0[20],15.0[2],8.0[25],8.0[25],3.0[54],3.0[54],null,null,"; - // 0,BPv[t]ofBucket1,TPv[t]ofBucket1,BPv[t]ofBucket2,TPv[t]ofBucket2,... + String res = "0.0[0],1.0[20],15.0[2],8.0[25],8.0[25],3.0[54],3.0[54],null,null,200.0[200],"; + // 0,P1v[P1t],BPv[t]ofBucket1,TPv[t]ofBucket1,BPv[t]ofBucket2,TPv[t]ofBucket2,...,Pnv[Pnt] try (Connection connection = DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); Statement statement = connection.createStatement()) { boolean hasResultSet = statement.execute( - "SELECT min_value(s0), max_value(s0)" + "SELECT min_value(s0), max_value(s0)" // do not change sequence + + ",min_time(s0), max_time(s0), first_value(s0), last_value(s0)" + " FROM root.vehicle.d0 group by ([0,100),25ms)"); Assert.assertTrue(hasResultSet); try (ResultSet resultSet = statement.getResultSet()) { @@ -157,16 +161,15 @@ public class MyTest_MinMax { public void test3() { prepareData3(); - // String[] res = new String[]{"0,1[10],10[2]", "25,2[40],8[30]", "50,4[72],20[62]", - // "75,1[90],11[80]"}; - String res = "1.0[10],10.0[2],2.0[40],8.0[30],4.0[72],20.0[62],1.0[90],11.0[80],"; - // 0,BPv[t]ofBucket1,TPv[t]ofBucket1,BPv[t]ofBucket2,TPv[t]ofBucket2,... + String res = + "0.0[0],1.0[10],10.0[2],2.0[40],8.0[30],4.0[72],20.0[62],1.0[90],11.0[80],200.0[200],"; try (Connection connection = DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); Statement statement = connection.createStatement()) { boolean hasResultSet = statement.execute( "SELECT min_value(s0), max_value(s0)" + + ",min_time(s0), max_time(s0), first_value(s0), last_value(s0)" + " FROM root.vehicle.d0 group by ([0,100),25ms)"); // don't change the // sequence!!! @@ -229,16 +232,14 @@ public class MyTest_MinMax { public void test3_2() { prepareData3_2(); - // String[] res = new String[]{"0,1[10],10[2]", "25,null,null", "50,4[72],20[62]", - // "75,1[90],11[80]"}; - String res = "1.0[10],10.0[2],null,null,4.0[72],20.0[62],1.0[90],11.0[80],"; - // 0,BPv[t]ofBucket1,TPv[t]ofBucket1,BPv[t]ofBucket2,TPv[t]ofBucket2,... + String res = "0.0[0],1.0[10],10.0[2],null,null,4.0[72],20.0[62],1.0[90],11.0[80],200.0[200],"; try (Connection connection = DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); Statement statement = connection.createStatement()) { boolean hasResultSet = statement.execute( "SELECT min_value(s0), max_value(s0)" + + ",min_time(s0), max_time(s0), first_value(s0), last_value(s0)" + " FROM root.vehicle.d0 group by ([0,100),25ms)"); // don't change the // sequence!!! diff --git a/server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_MinMaxLTTB.java b/server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_MinMaxLTTB.java index 3d3491cccb7..160df95c60d 100644 --- a/server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_MinMaxLTTB.java +++ b/server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_MinMaxLTTB.java @@ -108,6 +108,8 @@ public class MyTest_MinMaxLTTB { boolean hasResultSet = statement.execute( "SELECT min_value(s0), max_value(s0)" + // actually controlled by enableTri + + ",min_time(s0), max_time(s0), first_value(s0), last_value(s0)" + " FROM root.vehicle.d0 group by ([100,2100),250ms)"); // rps=4,nout=6,minmaxInterval=floor((tn-t2)/((nout-2)*rps/2))=250ms Assert.assertTrue(hasResultSet);
