This is an automated email from the ASF dual-hosted git repository. leirui pushed a commit to branch research/LTS-visualization in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit d1f7c6dd0759946aa3930479764dad5f7896705a Author: Lei Rui <[email protected]> AuthorDate: Sun Jan 28 09:03:48 2024 +0800 p1pnrps config --- .../resources/conf/iotdb-engine.properties | 8 + .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 47 ++++ .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 7 + .../groupby/GroupByWithoutValueFilterDataSet.java | 28 ++- .../db/integration/tri/MyTest_MinMaxLTTB.java | 271 +++++---------------- 5 files changed, 142 insertions(+), 219 deletions(-) diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties index 39258498cff..ebc535b6422 100644 --- a/server/src/assembly/resources/conf/iotdb-engine.properties +++ b/server/src/assembly/resources/conf/iotdb-engine.properties @@ -22,6 +22,14 @@ # MinMax, MinMaxLTTB, M4LTTB, LTTB, ILTS enable_Tri=MinMax +p1t=0 +p1v=0 + +pnt=100 +pnv=100 + +rps=4 + #################### ### enable CPV #################### diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 2bf7caaec93..92bb68f7f2f 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -435,6 +435,53 @@ public class IoTDBConfig { private String enableTri = ""; // MinMax, MinMaxLTTB, M4LTTB, LTTB, ILTS + private long p1t; + private double p1v; + private long pnt; + private double pnv; + + private int rps; + + public long getP1t() { + return p1t; + } + + public void setP1t(long p1t) { + this.p1t = p1t; + } + + public double getP1v() { + return p1v; + } + + public void setP1v(double p1v) { + this.p1v = p1v; + } + + public long getPnt() { + return pnt; + } + + public void setPnt(long pnt) { + this.pnt = pnt; + } + + public double getPnv() { + return pnv; + } + + public void setPnv(double pnv) { + this.pnv = pnv; + } + + public int getRps() { + return rps; + } + + public void setRps(int rps) { + this.rps = rps; + } + /** The display of stat performance interval in ms. */ private long performanceStatDisplayInterval = 60000; diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index e335d5ce0b9..f1523d8f739 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -540,6 +540,13 @@ public class IoTDBDescriptor { properties.getProperty("enable_CPV", Boolean.toString(conf.isEnableCPV())).trim())); conf.setEnableTri(properties.getProperty("enable_Tri", conf.getEnableTri()).trim()); + conf.setP1t(Long.parseLong(properties.getProperty("p1t", Long.toString(conf.getP1t())))); + conf.setP1v( + Double.parseDouble(properties.getProperty("p1v", Double.toString(conf.getP1v())))); + conf.setPnt(Long.parseLong(properties.getProperty("pnt", Long.toString(conf.getPnt())))); + conf.setPnv( + Double.parseDouble(properties.getProperty("pnv", Double.toString(conf.getPnv())))); + conf.setRps(Integer.parseInt(properties.getProperty("rps", Integer.toString(conf.getRps())))); conf.setPerformanceStatDisplayInterval( Long.parseLong( diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java index 71538e8c3dd..9bb07093067 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java +++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java @@ -153,16 +153,17 @@ public class GroupByWithoutValueFilterDataSet extends GroupByEngineDataSet { } public RowRecord nextWithoutConstraintTri_MinMaxLTTB() throws IOException { - RowRecord record; - // TODO tmp p1,pn,rps later passed by config - long p1t = 0; - double p1v = -1.2079272; - long pnt = 2100; - double pnv = -0.0211206; - int rps = 4; int divide = 2; // one LTTB bucket corresponds to rps/2 MinMax buckets + long p1t = CONFIG.getP1t(); + double p1v = CONFIG.getP1v(); + long pnt = CONFIG.getPnt(); + double pnv = CONFIG.getPnv(); + int rps = CONFIG.getRps(); + + RowRecord record; + // concat results into a string record = new RowRecord(0); StringBuilder series = new StringBuilder(); @@ -184,7 +185,7 @@ public class GroupByWithoutValueFilterDataSet extends GroupByEngineDataSet { // [0,3),[3,6),[6,9), no need incomplete [9,11) // then the number of buckets must be Math.floor((endTime-startTime)/interval) localCurStartTime += interval) { - System.out.println(localCurStartTime); + // System.out.println(localCurStartTime); // not change real curStartTime&curEndTime // attention the returned aggregations need deep copy if using directly List<AggregateResult> aggregations = @@ -204,8 +205,8 @@ public class GroupByWithoutValueFilterDataSet extends GroupByEngineDataSet { } // Second step: apply LTTB on the MinMax preselection result - System.out.println(times); - System.out.println(values); + // System.out.println(times); + // System.out.println(values); int N1 = (int) Math.floor((endTime * 1.0 - startTime) / interval); // MinMax桶数 int N2 = N1 / (rps / divide); // LTTB桶数 // 全局首点 @@ -266,7 +267,8 @@ public class GroupByWithoutValueFilterDataSet extends GroupByEngineDataSet { long t = times.get(j); double v = (double) values.get(j); // TODO double area = IOMonitor2.calculateTri(lt, lv, t, v, rt, rv); - System.out.printf("curr=%d,t=%d,area=%f,lt=%d%n", currentBucket, t, area, lt); + // System.out.printf("curr=%d,t=%d,area=%f,lt=%d%n", currentBucket, t, area, + // lt); if (area > maxArea) { maxArea = area; select_t = t; @@ -285,7 +287,7 @@ public class GroupByWithoutValueFilterDataSet extends GroupByEngineDataSet { lv = select_v; } - // 下面处理最后一个桶 + // ==========下面处理最后一个桶=============== // 现在找到当前非空桶里距离lr垂直距离最远的点 double maxArea = -1; long select_t = -1; @@ -296,7 +298,7 @@ public class GroupByWithoutValueFilterDataSet extends GroupByEngineDataSet { long t = times.get(j); double v = (double) values.get(j); // TODO double area = IOMonitor2.calculateTri(lt, lv, t, v, pnt, pnv); // 全局尾点作为右边固定点 - System.out.printf("curr=%d,t=%d,area=%f,lt=%d%n", currentBucket, t, area, lt); + // System.out.printf("curr=%d,t=%d,area=%f,lt=%d%n", currentBucket, t, area, lt); if (area > maxArea) { maxArea = area; select_t = t; diff --git a/server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_MinMaxLTTB.java b/server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_MinMaxLTTB.java index e80a6c3f8cf..8ee541a2514 100644 --- a/server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_MinMaxLTTB.java +++ b/server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_MinMaxLTTB.java @@ -24,7 +24,6 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.compaction.CompactionStrategy; import org.apache.iotdb.db.utils.EnvironmentUtils; import org.apache.iotdb.jdbc.Config; -import org.apache.iotdb.jdbc.IoTDBStatement; import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; import org.junit.After; @@ -73,13 +72,12 @@ public class MyTest_MinMaxLTTB { config.setCompactionStrategy(CompactionStrategy.NO_COMPACTION); config.setEnableTri("MinMaxLTTB"); + // config.setP1t(0); + // config.setP1v(-1.2079272); + // config.setPnt(2100); + // config.setPnv(-0.0211206); + // config.setRps(4); - // 但是如果走的是unpackOneChunkMetaData(firstChunkMetadata)就没问题， - // 因为它直接用chunk元数据去构造pageReader， - // 但是如果走的是传统聚合类型->seriesAggregateReader->seriesReader->hasNextOverlappedPage里 - // cachedBatchData = BatchDataFactory.createBatchData(dataType, orderUtils.getAscending(), true) - // 这个路径就错了，把聚合类型赋给batchData了。所以这个LocalGroupByExecutor bug得在有overlap数据的时候才能复现 - // （那刚好我本文数据都不会有Overlap，可以用LocalGroupByExecutor来得到正确结果） config.setEnableCPV(false); TSFileDescriptor.getInstance().getConfig().setEnableMinMaxLSM(false); TSFileDescriptor.getInstance().getConfig().setUseStatistics(false); @@ -93,64 +91,70 @@ public class MyTest_MinMaxLTTB { EnvironmentUtils.cleanEnv(); } - @Test - public void test1() throws Exception { - prepareData1(); - String res = "0,1[20],15[2],8[25],8[25],3[54],3[54],null,null,"; - try (Connection connection = - DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement()) { - boolean hasResultSet = - statement.execute( - "SELECT min_value(s0), max_value(s0)" - + " FROM root.vehicle.d0 group by ([0,100),25ms)"); - Assert.assertTrue(hasResultSet); - try (ResultSet resultSet = statement.getResultSet()) { - int i = 0; - while (resultSet.next()) { - // 注意从1开始编号，所以第一列是无意义时间戳 - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(2); - System.out.println(ans); - Assert.assertEquals(res, ans); - } - } - System.out.println(((IoTDBStatement) statement).executeFinish()); - } catch (Exception e) { - e.printStackTrace(); - fail(e.getMessage()); - } - } - - private static void prepareData1() { - // data: - // https://user-images.githubusercontent.com/33376433/151985070-73158010-8ba0-409d-a1c1-df69bad1aaee.png - // only first chunk - // no overlap, no delete - try (Connection connection = - DriverManager.getConnection( - Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement()) { - - for (String sql : creationSqls) { - statement.execute(sql); - } - - statement.execute(String.format(Locale.ENGLISH, insertTemplate, 1, 5.0)); - statement.execute(String.format(Locale.ENGLISH, insertTemplate, 2, 15.0)); - statement.execute(String.format(Locale.ENGLISH, insertTemplate, 20, 1.0)); - statement.execute(String.format(Locale.ENGLISH, insertTemplate, 25, 8.0)); - statement.execute(String.format(Locale.ENGLISH, insertTemplate, 54, 3.0)); - statement.execute(String.format(Locale.ENGLISH, insertTemplate, 120, 8.0)); - statement.execute("FLUSH"); - - } catch (Exception e) { - e.printStackTrace(); - } - } + // @Test + // public void test1() throws Exception { + // prepareData1(); + // String res = "0,1[20],15[2],8[25],8[25],3[54],3[54],null,null,"; + // try (Connection connection = + // DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); + // Statement statement = connection.createStatement()) { + // boolean hasResultSet = + // statement.execute( + // "SELECT min_value(s0), max_value(s0)" + // + " FROM root.vehicle.d0 group by ([0,100),25ms)"); + // Assert.assertTrue(hasResultSet); + // try (ResultSet resultSet = statement.getResultSet()) { + // int i = 0; + // while (resultSet.next()) { + // // 注意从1开始编号，所以第一列是无意义时间戳 + // String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(2); + // System.out.println(ans); + // Assert.assertEquals(res, ans); + // } + // } + // System.out.println(((IoTDBStatement) statement).executeFinish()); + // } catch (Exception e) { + // e.printStackTrace(); + // fail(e.getMessage()); + // } + // } + // + // private static void prepareData1() { + // // data: + // // + // https://user-images.githubusercontent.com/33376433/151985070-73158010-8ba0-409d-a1c1-df69bad1aaee.png + // // only first chunk + // // no overlap, no delete + // try (Connection connection = + // DriverManager.getConnection( + // Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); + // Statement statement = connection.createStatement()) { + // + // for (String sql : creationSqls) { + // statement.execute(sql); + // } + // + // statement.execute(String.format(Locale.ENGLISH, insertTemplate, 1, 5.0)); + // statement.execute(String.format(Locale.ENGLISH, insertTemplate, 2, 15.0)); + // statement.execute(String.format(Locale.ENGLISH, insertTemplate, 20, 1.0)); + // statement.execute(String.format(Locale.ENGLISH, insertTemplate, 25, 8.0)); + // statement.execute(String.format(Locale.ENGLISH, insertTemplate, 54, 3.0)); + // statement.execute(String.format(Locale.ENGLISH, insertTemplate, 120, 8.0)); + // statement.execute("FLUSH"); + // + // } catch (Exception e) { + // e.printStackTrace(); + // } + // } @Test public void test2() throws Exception { prepareData2(); + config.setP1t(0); + config.setP1v(-1.2079272); + config.setPnt(2100); + config.setPnv(-0.0211206); + config.setRps(4); String res = "-1.2079272[0],1.101946[200],-1.014322[700],0.809559[1500],-0.785419[1600],-0.0211206[2100],"; try (Connection connection = @@ -229,149 +233,4 @@ public class MyTest_MinMaxLTTB { e.printStackTrace(); } } - - @Test - public void test3() { - prepareData3(); - - // String[] res = new String[]{"0,1[10],10[2]", "25,2[40],8[30]", "50,4[72],20[62]", - // "75,1[90],11[80]"}; - String res = "0,1[10],10[2],2[40],8[30],4[72],20[62],1[90],11[80],"; - // 0,BPv[t]ofBucket1,TPv[t]ofBucket1,BPv[t]ofBucket2,TPv[t]ofBucket2,... - try (Connection connection = - DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement()) { - boolean hasResultSet = - statement.execute( - "SELECT min_value(s0), max_value(s0)" - + " FROM root.vehicle.d0 group by ([0,100),25ms)"); // don't change the - // sequence!!! - - Assert.assertTrue(hasResultSet); - try (ResultSet resultSet = statement.getResultSet()) { - int i = 0; - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(2); - System.out.println(ans); - Assert.assertEquals(res, ans); - } - } - } catch (Exception e) { - e.printStackTrace(); - fail(e.getMessage()); - } - } - - private static void prepareData3() { - // data: - // https://user-images.githubusercontent.com/33376433/152003603-6b4e7494-00ff-47e4-bf6e-cab3c8600ce2.png - try (Connection connection = - DriverManager.getConnection( - Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement()) { - - for (String sql : creationSqls) { - statement.execute(sql); - } - - statement.execute(String.format(Locale.ENGLISH, insertTemplate, 1, 5)); - statement.execute(String.format(Locale.ENGLISH, insertTemplate, 2, 10)); - statement.execute(String.format(Locale.ENGLISH, insertTemplate, 10, 1)); - statement.execute(String.format(Locale.ENGLISH, insertTemplate, 20, 5)); - statement.execute("FLUSH"); - - statement.execute(String.format(Locale.ENGLISH, insertTemplate, 22, 4)); - statement.execute(String.format(Locale.ENGLISH, insertTemplate, 30, 8)); - statement.execute(String.format(Locale.ENGLISH, insertTemplate, 40, 2)); - statement.execute(String.format(Locale.ENGLISH, insertTemplate, 55, 5)); - statement.execute("FLUSH"); - - statement.execute(String.format(Locale.ENGLISH, insertTemplate, 60, 15)); - statement.execute(String.format(Locale.ENGLISH, insertTemplate, 62, 20)); - statement.execute(String.format(Locale.ENGLISH, insertTemplate, 65, 8)); - statement.execute(String.format(Locale.ENGLISH, insertTemplate, 70, 18)); - statement.execute("FLUSH"); - - statement.execute(String.format(Locale.ENGLISH, insertTemplate, 72, 4)); - statement.execute(String.format(Locale.ENGLISH, insertTemplate, 80, 11)); - statement.execute(String.format(Locale.ENGLISH, insertTemplate, 90, 1)); - statement.execute(String.format(Locale.ENGLISH, insertTemplate, 105, 7)); - statement.execute("FLUSH"); - } catch (Exception e) { - e.printStackTrace(); - } - } - - @Test - public void test3_2() { - prepareData3_2(); - - // String[] res = new String[]{"0,1[10],10[2]", "25,null,null", "50,4[72],20[62]", - // "75,1[90],11[80]"}; - String res = "0,1[10],10[2],null,null,4[72],20[62],1[90],11[80],"; - // 0,BPv[t]ofBucket1,TPv[t]ofBucket1,BPv[t]ofBucket2,TPv[t]ofBucket2,... - try (Connection connection = - DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement()) { - boolean hasResultSet = - statement.execute( - "SELECT min_value(s0), max_value(s0)" - + " FROM root.vehicle.d0 group by ([0,100),25ms)"); // don't change the - // sequence!!! - - Assert.assertTrue(hasResultSet); - try (ResultSet resultSet = statement.getResultSet()) { - int i = 0; - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(2); - System.out.println(ans); - Assert.assertEquals(res, ans); - } - } - } catch (Exception e) { - e.printStackTrace(); - fail(e.getMessage()); - } - } - - private static void prepareData3_2() { - // data: - // https://user-images.githubusercontent.com/33376433/152003603-6b4e7494-00ff-47e4-bf6e-cab3c8600ce2.png - // remove chunk 2 so that bucket 2 is empty - try (Connection connection = - DriverManager.getConnection( - Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement()) { - - for (String sql : creationSqls) { - statement.execute(sql); - } - - statement.execute(String.format(Locale.ENGLISH, insertTemplate, 1, 5)); - statement.execute(String.format(Locale.ENGLISH, insertTemplate, 2, 10)); - statement.execute(String.format(Locale.ENGLISH, insertTemplate, 10, 1)); - statement.execute(String.format(Locale.ENGLISH, insertTemplate, 20, 5)); - statement.execute("FLUSH"); - - // statement.execute(String.format(Locale.ENGLISH, insertTemplate, 22, 4)); - // statement.execute(String.format(Locale.ENGLISH, insertTemplate, 30, 8)); - // statement.execute(String.format(Locale.ENGLISH, insertTemplate, 40, 2)); - // statement.execute(String.format(Locale.ENGLISH, insertTemplate, 55, 5)); - // statement.execute("FLUSH"); - - statement.execute(String.format(Locale.ENGLISH, insertTemplate, 60, 15)); - statement.execute(String.format(Locale.ENGLISH, insertTemplate, 62, 20)); - statement.execute(String.format(Locale.ENGLISH, insertTemplate, 65, 8)); - statement.execute(String.format(Locale.ENGLISH, insertTemplate, 70, 18)); - statement.execute("FLUSH"); - - statement.execute(String.format(Locale.ENGLISH, insertTemplate, 72, 4)); - statement.execute(String.format(Locale.ENGLISH, insertTemplate, 80, 11)); - statement.execute(String.format(Locale.ENGLISH, insertTemplate, 90, 1)); - statement.execute(String.format(Locale.ENGLISH, insertTemplate, 105, 7)); - statement.execute("FLUSH"); - } catch (Exception e) { - e.printStackTrace(); - } - } }
