This is an automated email from the ASF dual-hosted git repository. leirui pushed a commit to branch research/LTS-visualization in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 2213045a092412c75d9e8d9b16a9d7e34411922e Author: Lei Rui <[email protected]> AuthorDate: Wed Jan 31 21:22:02 2024 +0800 skip repeat iter; test convex prune points --- .../resources/conf/iotdb-engine.properties | 1 + .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 9 +++ .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 3 + .../groupby/LocalGroupByExecutorTri_ILTS.java | 68 ++++++++++++++++++---- .../iotdb/db/integration/tri/MyTest_ILTS.java | 5 +- 5 files changed, 73 insertions(+), 13 deletions(-) diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties index 7ec63153d1c..1e1306ddc67 100644 --- a/server/src/assembly/resources/conf/iotdb-engine.properties +++ b/server/src/assembly/resources/conf/iotdb-engine.properties @@ -37,6 +37,7 @@ numIterations=4 acc_avg=false acc_rectangle=true acc_convex=false +acc_iterRepeat=true #################### ### enable CPV diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 9d749c88d3b..04eba688ecc 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -479,6 +479,15 @@ public class IoTDBConfig { private boolean acc_avg = true; private boolean acc_rectangle = true; private boolean acc_convex = false; + private boolean acc_iterRepeat = true; + + public boolean isAcc_iterRepeat() { + return acc_iterRepeat; + } + + public void setAcc_iterRepeat(boolean acc_iterRepeat) { + this.acc_iterRepeat = acc_iterRepeat; + } public long getP1t() { return p1t; diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 96b4d496c01..afe5e484119 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -559,6 +559,9 @@ public class IoTDBDescriptor { conf.setAcc_convex( Boolean.parseBoolean( properties.getProperty("acc_convex", Boolean.toString(conf.isAcc_convex())))); + conf.setAcc_iterRepeat( + Boolean.parseBoolean( + properties.getProperty("acc_iterRepeat", Boolean.toString(conf.isAcc_iterRepeat())))); conf.setPerformanceStatDisplayInterval( Long.parseLong( diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_ILTS.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_ILTS.java index 81f9af04262..12a8b9b9cbf 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_ILTS.java +++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_ILTS.java @@ -176,19 +176,37 @@ public class LocalGroupByExecutorTri_ILTS implements GroupByExecutor { result.reset(); } - long[] lastIter_t = new long[N1]; // N1不包括全局首尾点 + long[] lastIter_t = new long[N1]; // N1不包括全局首尾点，初始化都是0，假设真实时间戳都大于0 double[] lastIter_v = new double[N1]; // N1不包括全局首尾点 - for (int num = 0; num < numIterations; num++) { + + // TODO: 如果和上次迭代时使用的lr一样那么这个bucket这次迭代就使用上次的采点结果，不必重复计算 + boolean[] needRecalc = new boolean[N1]; // N1不包括全局首尾点，初始化都是false + + int num = 0; // 注意从0开始！ + for (; num < numIterations; num++) { // NOTE: init lt&lv at the start of each iteration is a must, because they are modified in // each iteration lt = CONFIG.getP1t(); lv = CONFIG.getP1v(); - StringBuilder series = new StringBuilder(); // TODO debug - // 全局首点 - series.append(p1v).append("[").append(p1t).append("]").append(","); // TODO debug + boolean[] currentNeedRecalc = new boolean[N1]; // N1不包括全局首尾点，初始化都是false + boolean allFalseFlag = true; // 如果非首轮迭代全部都是false那可以提前结束迭代，因为后面都不会再有任何变化 + + // StringBuilder series = new StringBuilder(); // TODO debug + // // 全局首点 + // series.append(p1v).append("[").append(p1t).append("]").append(","); // TODO debug + // 遍历分桶 Assume no empty buckets for (int b = 0; b < N1; b++) { + if (CONFIG.isAcc_iterRepeat() && num > 0 && !needRecalc[b]) { + // 排除num=0，因为第一次迭代要全部算的 + // 不需要更新本轮迭代本桶选点，或者说本轮迭代本桶选点就是lastIter内已有结果 + // 也不需要更新currentNeedRecalc + // 下一个桶自然地以select_t, select_v作为左桶固定点 + lt = lastIter_t[b]; + lv = lastIter_v[b]; + continue; + } double rt = 0; // must initialize as zero, because may be used as sum for average double rv = 0; // must initialize as zero, because may be used as sum for average // 计算右边桶的固定点 @@ -217,7 +235,6 @@ public class LocalGroupByExecutorTri_ILTS implements GroupByExecutor { if (CONFIG.isAcc_avg()) { if (chunkSuit4Tri.chunkMetadata.getStartTime() >= rightStartTime && chunkSuit4Tri.chunkMetadata.getEndTime() < rightEndTime) { - System.out.println("herehere"); // TODO 以后元数据可以增加sum of timestamps，目前就基于时间戳均匀间隔1的假设来处理 rt += (chunkSuit4Tri.chunkMetadata.getStartTime() @@ -378,6 +395,11 @@ public class LocalGroupByExecutorTri_ILTS implements GroupByExecutor { maxDistance = distance; select_t = timestamp; select_v = v; + + // TODO 下面假装已经有凸包剪枝，先实验看看如果跳过一些点不用遍历有多少加速效果 + if (CONFIG.isAcc_convex()) { + break; + } } } } @@ -391,8 +413,22 @@ public class LocalGroupByExecutorTri_ILTS implements GroupByExecutor { // 还要注意的是如果被rectangle提前剪枝掉了就不会走到这一步，也就是说那个chunk的pageReader可能还留着 } } - // 记录结果 // TODO debug - series.append(select_v).append("[").append(select_t).append("]").append(","); + + // // 记录结果 // TODO debug + // series.append(select_v).append("[").append(select_t).append("]").append(","); + + // 更新currentNeedRecalc,注意在记录本轮迭代本桶选点之前判断 + if (CONFIG.isAcc_iterRepeat() && select_t != lastIter_t[b]) { // 本次迭代选点结果和上一轮不一样 + allFalseFlag = false; + if (b == 0) { // 第一个桶 + currentNeedRecalc[b + 1] = true; // 作为右边桶的左边固定点变了，所以下一轮右边桶要重新采点 + } else if (b == N1 - 1) { // 最后一个桶 + currentNeedRecalc[b - 1] = true; // 作为左边桶的右边固定点变了，所以下一轮左边桶要重新采点 + } else { + currentNeedRecalc[b - 1] = true; // 作为左边桶的右边固定点变了，所以下一轮左边桶要重新采点 + currentNeedRecalc[b + 1] = true; // 作为右边桶的左边固定点变了，所以下一轮右边桶要重新采点 + } + } // 更新lt,lv // 下一个桶自然地以select_t, select_v作为左桶固定点 @@ -403,10 +439,20 @@ public class LocalGroupByExecutorTri_ILTS implements GroupByExecutor { lastIter_v[b] = select_v; } // 遍历分桶结束 - // 全局尾点 // TODO debug - series.append(pnv).append("[").append(pnt).append("]").append(","); - System.out.println(series); + // // 全局尾点 // TODO debug + // series.append(pnv).append("[").append(pnt).append("]").append(","); + // System.out.println(series); + + if (CONFIG.isAcc_iterRepeat() && allFalseFlag) { + num++; // +1表示是完成的迭代次数 + break; + } + // 否则currentNeedRecalc里至少有一个true，因此继续迭代 + needRecalc = currentNeedRecalc; + // System.out.println(Arrays.toString(needRecalc)); // TODO debug + } // end Iterations + // System.out.println("number of iterations=" + num); // TODO debug // 全局首点 series_final.append(p1v).append("[").append(p1t).append("]").append(","); diff --git a/server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_ILTS.java b/server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_ILTS.java index ccef5e9fa84..1e8e1f47180 100644 --- a/server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_ILTS.java +++ b/server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_ILTS.java @@ -80,7 +80,8 @@ public class MyTest_ILTS { // config.setNumIterations(4); config.setAcc_avg(true); config.setAcc_rectangle(true); - config.setAcc_convex(false); + config.setAcc_convex(true); + config.setAcc_iterRepeat(true); config.setEnableCPV(false); TSFileDescriptor.getInstance().getConfig().setEnableMinMaxLSM(false); @@ -226,7 +227,7 @@ public class MyTest_ILTS { @Test public void test2() { prepareData2(); - config.setNumIterations(4); + config.setNumIterations(8); String res = "-1.2079272[0],1.101946[200],-0.523204[300],0.145359[500],-1.014322[700]," + "0.532565[900],-0.122525[1200],-0.676077[1300],0.809559[1500],0.315869[1800],"
