This is an automated email from the ASF dual-hosted git repository. leirui pushed a commit to branch research/LTS-visualization in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit fc7fcb5d61ec2b797c9456e0ada9d41b012f260f Author: Lei Rui <[email protected]> AuthorDate: Mon Oct 7 22:03:56 2024 +0800 add --- .../resources/conf/iotdb-engine.properties | 3 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 9 ++ .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 4 + .../groupby/LocalGroupByExecutorTri_ILTS.java | 69 ++++++++- .../iotdb/db/integration/tri/MyTest_ILTS_2.java | 159 +++++++++++++++++++++ 5 files changed, 237 insertions(+), 7 deletions(-) diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties index b88b4557743..d68392e0e9d 100644 --- a/server/src/assembly/resources/conf/iotdb-engine.properties +++ b/server/src/assembly/resources/conf/iotdb-engine.properties @@ -25,7 +25,8 @@ enable_Tri="" # segment error threshold for SimPiece, SC, FSW epsilon=100 -#for +auto_p1n=false + p1t=0 p1v=0 diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 1bd01c3a5f5..b9e25221d8b 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -437,6 +437,7 @@ public class IoTDBConfig { private double epsilon = 100; // for SimPiece + private boolean autoP1n = true; private long p1t; private double p1v; private long pnt; @@ -491,6 +492,14 @@ public class IoTDBConfig { this.acc_iterRepeat = acc_iterRepeat; } + public boolean getAutoP1n() { + return autoP1n; + } + + public void setAutoP1n(boolean autoP1n) { + this.autoP1n = autoP1n; + } + public long getP1t() { return p1t; } diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 70c573e1e1e..10993aa53df 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -543,6 +543,10 @@ public class IoTDBDescriptor { conf.setEpsilon( Double.parseDouble( properties.getProperty("epsilon", Double.toString(conf.getEpsilon())))); + + conf.setAutoP1n( + Boolean.parseBoolean( + properties.getProperty("auto_p1n", Boolean.toString(conf.getAutoP1n())).trim())); conf.setP1t(Long.parseLong(properties.getProperty("p1t", Long.toString(conf.getP1t())))); conf.setP1v( Double.parseDouble(properties.getProperty("p1v", Double.toString(conf.getP1v())))); diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_ILTS.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_ILTS.java index 26a397ba229..4312cfb49fa 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_ILTS.java +++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_ILTS.java @@ -48,6 +48,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.BitSet; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -63,10 +64,10 @@ public class LocalGroupByExecutorTri_ILTS implements GroupByExecutor { // keys: 0,1,...,(int) Math.floor((endTime * 1.0 - startTime) / interval)-1 private Map<Integer, List<ChunkSuit4Tri>> splitChunkList = new HashMap<>(); - private final long p1t = CONFIG.getP1t(); - private final double p1v = CONFIG.getP1v(); - private final long pnt = CONFIG.getPnt(); - private final double pnv = CONFIG.getPnv(); + private long p1t; + private double p1v; + private long pnt; + private double pnv; private long lt; private double lv; @@ -136,6 +137,56 @@ public class LocalGroupByExecutorTri_ILTS implements GroupByExecutor { } } + if (CONFIG.getAutoP1n()) { + // get real p1 + List<ChunkSuit4Tri> firstBucket = splitChunkList.get(0); + sortByStartTime(firstBucket); + ChunkSuit4Tri firstChunk = firstBucket.get(0); + if (firstChunk.pageReader == null) { + firstChunk.pageReader = + FileLoaderUtils.loadPageReaderList4CPV(firstChunk.chunkMetadata, this.timeFilter); + } + PageReader pageReader = firstChunk.pageReader; + for (int j = 0; j < firstChunk.chunkMetadata.getStatistics().getCount(); j++) { + long timestamp = pageReader.timeBuffer.getLong(j * 8); + if (timestamp < startTime) { + continue; + } else if (timestamp >= startTime) { + ByteBuffer valueBuffer = pageReader.valueBuffer; + double v = valueBuffer.getDouble(pageReader.timeBufferLength + j * 8); + p1t = timestamp; + p1v = v; + break; + } + } + + // get real pn + List<ChunkSuit4Tri> lastBucket = splitChunkList.get(N1 - 1); + sortByStartTime(lastBucket); + ChunkSuit4Tri lastChunk = lastBucket.get(lastBucket.size() - 1); + if (lastChunk.pageReader == null) { + lastChunk.pageReader = + FileLoaderUtils.loadPageReaderList4CPV(lastChunk.chunkMetadata, this.timeFilter); + } + pageReader = lastChunk.pageReader; + for (int j = 0; j < lastChunk.chunkMetadata.getStatistics().getCount(); j++) { + long timestamp = pageReader.timeBuffer.getLong(j * 8); + if (timestamp > endTime) { // pn can be at endTime + break; + } else { + ByteBuffer valueBuffer = pageReader.valueBuffer; + double v = valueBuffer.getDouble(pageReader.timeBufferLength + j * 8); + pnt = timestamp; + pnv = v; + } + } + } else { + p1t = CONFIG.getP1t(); + p1v = CONFIG.getP1v(); + pnt = CONFIG.getPnt(); + pnv = CONFIG.getPnv(); + } + } catch (IOException e) { throw new QueryProcessException(e.getMessage()); } @@ -144,6 +195,12 @@ public class LocalGroupByExecutorTri_ILTS implements GroupByExecutor { // start); } + private static void sortByStartTime(List<ChunkSuit4Tri> list) { + Collections.sort( + list, + (a, b) -> Long.compare(a.chunkMetadata.getStartTime(), b.chunkMetadata.getStartTime())); + } + @Override public void addAggregateResult(AggregateResult aggrResult) { results.add(aggrResult); @@ -170,8 +227,8 @@ public class LocalGroupByExecutorTri_ILTS implements GroupByExecutor { for (; num < numIterations; num++) { // NOTE: init lt&lv at the start of each iteration is a must, because they are modified in // each iteration - lt = CONFIG.getP1t(); - lv = CONFIG.getP1v(); + lt = p1t; + lv = p1v; boolean allSameFlag = true; boolean currentLeftSame = true; diff --git a/server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_ILTS_2.java b/server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_ILTS_2.java new file mode 100644 index 00000000000..37fc35f5914 --- /dev/null +++ b/server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_ILTS_2.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.integration.tri; + +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.engine.compaction.CompactionStrategy; +import org.apache.iotdb.db.utils.EnvironmentUtils; +import org.apache.iotdb.jdbc.Config; +import org.apache.iotdb.jdbc.IoTDBStatement; +import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.Locale; + +import static org.junit.Assert.fail; + +public class MyTest_ILTS_2 { + + /* + * Requirements: + * (1) Don't change the sequence of the above two aggregates + * (2) Assume each chunk has only one page. + * (3) Assume all chunks are sequential and no deletes. + * (4) Assume plain encoding, UNCOMPRESSED, Long or Double data type, no compaction + * (5) Assume no empty bucket + */ + private static final String TIMESTAMP_STR = "Time"; + + private static String[] creationSqls = + new String[] { + "SET STORAGE GROUP TO root.vehicle.d0", + "CREATE TIMESERIES root.vehicle.d0.s0 WITH DATATYPE=DOUBLE, ENCODING=PLAIN", + // IoTDB int data type does not support plain encoding, so use long data type + }; + + private final String d0s0 = "root.vehicle.d0.s0"; + + private static final String insertTemplate = + "INSERT INTO root.vehicle.d0(timestamp,s0)" + " VALUES(%d,%f)"; + + private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + + @Before + public void setUp() throws Exception { + TSFileDescriptor.getInstance().getConfig().setTimeEncoder("PLAIN"); + config.setTimestampPrecision("ms"); + config.setCompactionStrategy(CompactionStrategy.NO_COMPACTION); + + config.setEnableTri("ILTS"); + config.setAcc_avg(true); + config.setAcc_rectangle(true); + config.setAcc_convex(true); + config.setAcc_iterRepeat(true); + TSFileDescriptor.getInstance().getConfig().setWriteConvexHull(true); + + config.setEnableCPV(false); + TSFileDescriptor.getInstance().getConfig().setEnableMinMaxLSM(false); + TSFileDescriptor.getInstance().getConfig().setUseStatistics(false); + + EnvironmentUtils.envSetUp(); + Class.forName(Config.JDBC_DRIVER_NAME); + } + + @After + public void tearDown() throws Exception { + EnvironmentUtils.cleanEnv(); + } + + @Test + public void test1() { + prepareData1(); + config.setNumIterations(8); + String res = "10.0[2],1.0[10],8.0[30],5.0[55],20.0[62],1.0[90],7.0[102],"; + try (Connection connection = + DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { + boolean hasResultSet = + statement.execute( + "SELECT min_value(s0)" + // TODO not real min_value here, actually controlled by enableTri + + ",max_value(s0),min_time(s0), max_time(s0), first_value(s0), last_value(s0)" + + " FROM root.vehicle.d0 group by ([2,102),20ms)"); + // (102-2)/(7-2)=20ms + // note keep no empty buckets + + Assert.assertTrue(hasResultSet); + try (ResultSet resultSet = statement.getResultSet()) { + int i = 0; + while (resultSet.next()) { + String ans = resultSet.getString(2); + System.out.println(ans); + Assert.assertEquals(res, ans); + } + } + System.out.println(((IoTDBStatement) statement).executeFinish()); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + private static void prepareData1() { + // data: + // https://user-images.githubusercontent.com/33376433/152003603-6b4e7494-00ff-47e4-bf6e-cab3c8600ce2.png + // slightly modified + try (Connection connection = + DriverManager.getConnection( + Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { + + for (String sql : creationSqls) { + statement.execute(sql); + } + + long[] t = new long[] {1, 2, 10, 20, 22, 30, 40, 55, 60, 62, 65, 70, 72, 80, 90, 102}; + double[] v = new double[] {5, 10, 1, 5, 4, 8, 2, 5, 15, 20, 8, 18, 4, 11, 1, 7}; + config.setP1t(t[0]); + config.setP1v(v[0]); + config.setPnt(t[t.length - 1]); + config.setPnv(v[v.length - 1]); + + for (int i = 0; i < t.length; i++) { + statement.execute(String.format(Locale.ENGLISH, insertTemplate, t[i], v[i])); + if ((i + 1) % 4 == 0) { + statement.execute("FLUSH"); + } + } + statement.execute("FLUSH"); + } catch (Exception e) { + e.printStackTrace(); + } + } +}
