This is an automated email from the ASF dual-hosted git repository. leirui pushed a commit to branch research/LTS-visualization in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit e71099183e3ed7dbf978436601d67b6a66c37bc6 Author: Lei Rui <[email protected]> AuthorDate: Mon Sep 23 00:17:44 2024 +0800 test --- .../db/query/simpiece/MySample_fsw_full2.java | 216 ++++++++++++++++++++ .../simpiece/MySample_shrinkingcone_full2.java | 204 +++++++++++++++++++ .../db/query/simpiece/MySample_simpiece_full2.java | 226 +++++++++++++++++++++ 3 files changed, 646 insertions(+) diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_fsw_full2.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_fsw_full2.java new file mode 100644 index 00000000000..6f5ca979f51 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_fsw_full2.java @@ -0,0 +1,216 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Sim-Piece code forked from https://github.com/xkitsios/Sim-Piece.git + +package org.apache.iotdb.db.query.simpiece; + +import java.io.FileInputStream; +import java.io.FileWriter; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.List; + +public class MySample_fsw_full2 { + + public static void main(String[] args) { + String fileDir = "D:\\desktop\\NISTPV\\"; + String[] datasetNameList = new String[] {"WindSpeed", "Qloss", "Pyra1", "RTD"}; + int[] noutList = + new int[] { + 320, 400, 480, 580, 720, 960, 1200, 1600, 2000, 2400, 3000, 3600, 4000, 4400, 5000 + }; + + double[][] epsilonArray = { + { + 9.618271827697754, + 8.991525650024414, + 8.375, + 7.99461555480957, + 7.615596771240234, + 7.172431945800781, + 6.877520561218262, + 6.408135414123535, + 6.219999313354492, + 5.977571487426758, + 5.679183006286621, + 5.499468803405762, + 5.387459754943848, + 5.299999237060547, + 5.1635847091674805, + }, + { + 9.9945068359375E-4, 9.984970092773438E-4, 9.965896606445312E-4, 9.937286376953125E-4, + 9.899139404296875E-4, 9.813308715820312E-4, 9.6893310546875E-4, 9.42230224609375E-4, + 8.983612060546875E-4, 8.268356323242188E-4, 6.685256958007812E-4, 5.197525024414062E-4, + 4.99725341796875E-4, 4.987716674804688E-4, 4.8732757568359375E-4, + }, + { + 440.0235958099365, 423.5567502975464, 405.7711305618286, 396.31347465515137, + 380.6776990890503, 358.0119905471802, 336.5447692871094, 303.85207748413086, + 279.7541666030884, 257.3554916381836, 232.57367420196533, 211.35449981689453, + 200.9514446258545, 192.0128345489502, 178.28646087646484, + }, + { + 9.295397758483887, 7.5366668701171875, 6.473535537719727, 5.6244354248046875, + 4.713288307189941, 3.7258691787719727, 3.078885078430176, 2.4637460708618164, + 2.0368423461914062, 1.7683038711547852, 1.4741735458374023, 1.266657829284668, + 1.1668891906738281, 1.071258544921875, 0.9608230590820312, + } + }; + + // double[][] epsilonArray = new double[datasetNameList.length][]; + // for (int i = 0; i < datasetNameList.length; i++) { + // epsilonArray[i] = new double[noutList.length]; + // } + + for (int y = 0; y < datasetNameList.length; y++) { + String datasetName = datasetNameList[y]; + int start = 0; + int end = 1000_0000; + int N = end - start; + boolean hasHeader = false; + try (FileInputStream inputStream = new FileInputStream(fileDir + datasetName + ".csv")) { + String delimiter = ","; + TimeSeries ts = + TimeSeriesReader.getMyTimeSeries( + inputStream, delimiter, false, N, start, hasHeader, false); + for (int x = 0; x < noutList.length; x++) { + int nout = noutList[x]; + + // double epsilon = getFSWParam(nout, ts, 1e-6); + // epsilonArray[y][x] = epsilon; + + double epsilon = epsilonArray[y][x]; + + List<Point> reducedPoints = FSW.reducePoints(ts.data, epsilon); + System.out.println( + datasetName + + ": n=" + + N + + ",m=" + + nout + + ",epsilon=" + + epsilon + + ",actual m=" + + reducedPoints.size()); + try (PrintWriter writer = + new PrintWriter( + new FileWriter( + datasetName + + "-" + + N + + "-" + + nout + + "-" + + reducedPoints.size() + + "-fsw.csv"))) { + for (Point p : reducedPoints) { + writer.println(p.getTimestamp() + "," + p.getValue()); + } + } + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + for (int i = 0; i < epsilonArray.length; i++) { + for (int j = 0; j < epsilonArray[i].length; j++) { + System.out.print(epsilonArray[i][j] + ","); + } + System.out.println(); + } + } + + public static double getFSWParam(int nout, TimeSeries ts, double accuracy) throws IOException { + double epsilon = 1; + boolean directLess = false; + boolean directMore = false; + boolean skip = false; // TODO + int threshold = 2; // TODO + while (true) { + List<Point> reducedPoints = FSW.reducePoints(ts.data, epsilon); + if (reducedPoints.size() > nout) { + if (directMore) { + // TODO + if (Math.abs(reducedPoints.size() - nout) <= threshold) { + skip = true; + } + // + break; + } + if (!directLess) { + directLess = true; + } + epsilon *= 2; + } else { + if (directLess) { + // TODO + if (Math.abs(nout - reducedPoints.size()) <= threshold) { + skip = true; + } + // + break; + } + if (!directMore) { + directMore = true; + } + epsilon /= 2; + } + } + // TODO + if (skip) { + return epsilon; + } + // + // begin dichotomy + double left = 0; + double right = 0; + if (directLess) { + left = epsilon / 2; + right = epsilon; + } + if (directMore) { + left = epsilon; + right = epsilon * 2; + } + while (Math.abs(right - left) > accuracy) { + double mid = (left + right) / 2; + List<Point> reducedPoints = FSW.reducePoints(ts.data, mid); + if (reducedPoints.size() > nout) { + left = mid; + } else { + right = mid; + } + } + + // TODO + List<Point> reducedPoints = FSW.reducePoints(ts.data, left); + int n1 = reducedPoints.size(); + reducedPoints = FSW.reducePoints(ts.data, right); + int n2 = reducedPoints.size(); + if (Math.abs(n1 - nout) < Math.abs(n2 - nout)) { + return left; + } else { + return right; + } + // + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_shrinkingcone_full2.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_shrinkingcone_full2.java new file mode 100644 index 00000000000..e0fe728f52d --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_shrinkingcone_full2.java @@ -0,0 +1,204 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Sim-Piece code forked from https://github.com/xkitsios/Sim-Piece.git + +package org.apache.iotdb.db.query.simpiece; + +import java.io.FileInputStream; +import java.io.FileWriter; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.List; + +public class MySample_shrinkingcone_full2 { + + public static void main(String[] args) { + String fileDir = "D:\\desktop\\NISTPV\\"; + String[] datasetNameList = new String[] {"WindSpeed", "Qloss", "Pyra1", "RTD"}; + int[] noutList = + new int[] { + 320, 400, 480, 580, 720, 960, 1200, 1600, 2000, 2400, 3000, 3600, 4000, 4400, 5000 + }; + + double[][] epsilonArray = { + { + 14.388325691223145, 13.343345642089844, 12.697674751281738, 12.053359985351562, + 11.411823272705078, 10.700932502746582, 10.222302436828613, 9.640463829040527, + 9.169320106506348, 8.824727058410645, 8.389120101928711, 8.050561904907227, + 7.84444522857666, 7.675000190734863, 7.441162109375, + }, + { + 0.001, 0.001, 0.001, 0.001, + 0.001, 0.001, 0.001, 9.9945068359375E-4, + 9.9945068359375E-4, 9.9945068359375E-4, 9.984970092773438E-4, 9.946823120117188E-4, + 9.927749633789062E-4, 9.8419189453125E-4, 9.088516235351562E-4, + }, + { + 680.2196474075317, 650.261082649231, 620.9016036987305, 586.952600479126, + 547.2381553649902, 493.3626317977905, 453.35704135894775, 404.6129741668701, + 368.50129795074463, 342.09704971313477, 305.65501976013184, 276.0216178894043, + 257.88243865966797, 242.78428554534912, 223.49278926849365, + }, + { + 11.259881019592285, 9.402754783630371, 8.384316444396973, 7.238059043884277, + 5.794983863830566, 4.60251522064209, 3.8171539306640625, 3.0447826385498047, + 2.526477813720703, 2.1706619262695312, 1.816070556640625, 1.5562095642089844, + 1.4302129745483398, 1.3176708221435547, 1.1894960403442383, + } + }; + + // double[][] epsilonArray = new double[datasetNameList.length][]; + // for (int i = 0; i < datasetNameList.length; i++) { + // epsilonArray[i] = new double[noutList.length]; + // } + + for (int y = 0; y < datasetNameList.length; y++) { + String datasetName = datasetNameList[y]; + int start = 0; + int end = 1000_0000; + int N = end - start; + boolean hasHeader = false; + try (FileInputStream inputStream = new FileInputStream(fileDir + datasetName + ".csv")) { + String delimiter = ","; + TimeSeries ts = + TimeSeriesReader.getMyTimeSeries( + inputStream, delimiter, false, N, start, hasHeader, false); + for (int x = 0; x < noutList.length; x++) { + int nout = noutList[x]; + + // double epsilon = getSCParam(nout, ts, 1e-15); + // epsilonArray[y][x] = epsilon; + + double epsilon = epsilonArray[y][x]; + + List<Point> reducedPoints = ShrinkingCone.reducePoints(ts.data, epsilon); + System.out.println( + datasetName + + ": n=" + + N + + ",m=" + + nout + + ",epsilon=" + + epsilon + + ",actual m=" + + reducedPoints.size()); + try (PrintWriter writer = + new PrintWriter( + new FileWriter( + datasetName + + "-" + + N + + "-" + + nout + + "-" + + reducedPoints.size() + + "-sc.csv"))) { + for (Point p : reducedPoints) { + writer.println(p.getTimestamp() + "," + p.getValue()); + } + } + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + for (int i = 0; i < epsilonArray.length; i++) { // 遍历行 + for (int j = 0; j < epsilonArray[i].length; j++) { // 遍历列 + System.out.print(epsilonArray[i][j] + ","); + } + System.out.println(); + } + } + + public static double getSCParam(int nout, TimeSeries ts, double accuracy) throws IOException { + double epsilon = 1; + boolean directLess = false; + boolean directMore = false; + boolean skip = false; // TODO + int threshold = 2; // TODO + while (true) { + List<Point> reducedPoints = ShrinkingCone.reducePoints(ts.data, epsilon); + if (reducedPoints.size() > nout) { + if (directMore) { + // TODO + if (Math.abs(reducedPoints.size() - nout) <= threshold) { + skip = true; + } + // + break; + } + if (!directLess) { + directLess = true; + } + epsilon *= 2; + } else { + if (directLess) { + // TODO + if (Math.abs(nout - reducedPoints.size()) <= threshold) { + skip = true; + } + // + break; + } + if (!directMore) { + directMore = true; + } + epsilon /= 2; + } + } + // TODO + if (skip) { + return epsilon; + } + // + // begin dichotomy + double left = 0; + double right = 0; + if (directLess) { + left = epsilon / 2; + right = epsilon; + } + if (directMore) { + left = epsilon; + right = epsilon * 2; + } + while (Math.abs(right - left) > accuracy) { + double mid = (left + right) / 2; + List<Point> reducedPoints = ShrinkingCone.reducePoints(ts.data, mid); + if (reducedPoints.size() > nout) { + left = mid; + } else { + right = mid; + } + } + // TODO + List<Point> reducedPoints = FSW.reducePoints(ts.data, left); + int n1 = reducedPoints.size(); + reducedPoints = FSW.reducePoints(ts.data, right); + int n2 = reducedPoints.size(); + if (Math.abs(n1 - nout) < Math.abs(n2 - nout)) { + return left; + } else { + return right; + } + // + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_simpiece_full2.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_simpiece_full2.java new file mode 100644 index 00000000000..e0bdd3aab25 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_simpiece_full2.java @@ -0,0 +1,226 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Sim-Piece code forked from https://github.com/xkitsios/Sim-Piece.git + +package org.apache.iotdb.db.query.simpiece; + +import java.io.FileInputStream; +import java.io.FileWriter; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.Comparator; +import java.util.List; + +public class MySample_simpiece_full2 { + + public static void main(String[] args) { + String fileDir = "D:\\desktop\\NISTPV\\"; + String[] datasetNameList = new String[] {"WindSpeed", "Qloss", "Pyra1", "RTD"}; + int[] noutList = + new int[] { + 320, 400, 480, 580, 720, 960, 1200, 1600, 2000, 2400, 3000, 3600, 4000, 4400, 5000 + }; + + double[][] epsilonArray = { + { + 10.421952322125435, 9.571347452700138, 9.122375220060349, 8.709278352558613, + 8.077139288187027, 7.495633184909821, 7.083636365830898, 6.733693778514862, + 6.403274565935135, 6.160000003874302, 5.93758013099432, 5.787152819335461, + 5.647759035229683, 5.5742857083678246, 5.444876328110695, + }, + { + 9.987801313400269E-4, 9.798482060432434E-4, 9.232386946678162E-4, 8.238255977630615E-4, + 7.013455033302307E-4, 6.666630506515503E-4, 6.311237812042236E-4, 5.687475204467773E-4, + 5.405843257904053E-4, 5.235522985458374E-4, 5.015432834625244E-4, 4.99919056892395E-4, + 4.997774958610535E-4, 4.995688796043396E-4, 4.983171820640564E-4, + }, + { + 502.05052164942026, 485.91779258847237, 464.11991154402494, 436.75185588002205, + 411.17336819320917, 383.33036886900663, 366.5155046656728, 334.26181526482105, + 308.3470098376274, 288.7738408744335, 260.332594871521, 242.4736728295684, + 229.66440346837044, 219.4005109295249, 205.81142588704824, + }, + { + 13.637416116893291, 11.293272890150547, 9.741776660084724, 8.293842449784279, + 6.9908598735928535, 5.493516407907009, 4.693962275981903, 3.6259122267365456, + 2.9906081929802895, 2.5887924432754517, 2.1600354313850403, 1.8493054434657097, + 1.689537525177002, 1.5635392591357231, 1.3930133953690529, + } + }; + + // double[][] epsilonArray = new double[datasetNameList.length][]; + // for (int i = 0; i < datasetNameList.length; i++) { + // epsilonArray[i] = new double[noutList.length]; + // } + + for (int y = 0; y < datasetNameList.length; y++) { + String datasetName = datasetNameList[y]; + int start = 0; + int end = 1000_0000; + int N = end - start; + // apply Sim-Piece on the input file, outputting nout points saved in csvFile + boolean hasHeader = false; + try (FileInputStream inputStream = new FileInputStream(fileDir + datasetName + ".csv")) { + String delimiter = ","; + TimeSeries ts = + TimeSeriesReader.getMyTimeSeries( + inputStream, delimiter, false, N, start, hasHeader, false); + for (int x = 0; x < noutList.length; x++) { + int nout = noutList[x]; + // + // double epsilon = getSimPieceParam(nout, ts, 1e-8); + // epsilonArray[y][x] = epsilon; + + double epsilon = epsilonArray[y][x]; + + SimPiece simPiece = new SimPiece(ts.data, epsilon); + System.out.println( + datasetName + + ": n=" + + N + + ",m=" + + nout + + ",epsilon=" + + epsilon + + ",actual m=" + + simPiece.segments.size() * 2); + List<SimPieceSegment> segments = simPiece.segments; + segments.sort(Comparator.comparingLong(SimPieceSegment::getInitTimestamp)); + try (PrintWriter writer = + new PrintWriter( + new FileWriter( + datasetName + + "-" + + N + + "-" + + nout + + "-" + + segments.size() * 2 + + "-simpiece.csv"))) { + for (int i = 0; i < segments.size() - 1; i++) { + // start point of this segment + writer.println(segments.get(i).getInitTimestamp() + "," + segments.get(i).getB()); + // end point of this segment + double v = + (segments.get(i + 1).getInitTimestamp() - segments.get(i).getInitTimestamp()) + * segments.get(i).getA() + + segments.get(i).getB(); + writer.println(segments.get(i + 1).getInitTimestamp() + "," + v); + } + // the two end points of the last segment + writer.println( + segments.get(segments.size() - 1).getInitTimestamp() + + "," + + segments.get(segments.size() - 1).getB()); + double v = + (simPiece.lastTimeStamp - segments.get(segments.size() - 1).getInitTimestamp()) + * segments.get(segments.size() - 1).getA() + + segments.get(segments.size() - 1).getB(); + writer.println(simPiece.lastTimeStamp + "," + v); + } + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + for (int i = 0; i < epsilonArray.length; i++) { // 遍历行 + for (int j = 0; j < epsilonArray[i].length; j++) { // 遍历列 + System.out.print(epsilonArray[i][j] + ","); + } + System.out.println(); + } + } + + public static double getSimPieceParam(int nout, TimeSeries ts, double accuracy) + throws IOException { + double epsilon = 1; + boolean directLess = false; + boolean directMore = false; + boolean skip = false; // TODO + int threshold = 2; // TODO + while (true) { + SimPiece simPiece = new SimPiece(ts.data, epsilon); + if (simPiece.segments.size() * 2 > nout) { // note *2 for disjoint + if (directMore) { + // TODO + if (Math.abs(simPiece.segments.size() * 2 - nout) <= threshold) { + skip = true; + } + // + break; + } + if (!directLess) { + directLess = true; + } + epsilon *= 2; + } else { + if (directLess) { + // TODO + if (Math.abs(nout - simPiece.segments.size() * 2) <= threshold) { + skip = true; + } + // + break; + } + if (!directMore) { + directMore = true; + } + epsilon /= 2; + } + } + // TODO + if (skip) { + return epsilon; + } + // + // begin dichotomy + double left = 0; + double right = 0; + if (directLess) { + left = epsilon / 2; + right = epsilon; + } + if (directMore) { + left = epsilon; + right = epsilon * 2; + } + while (Math.abs(right - left) > accuracy) { + double mid = (left + right) / 2; + SimPiece simPiece = new SimPiece(ts.data, mid); + if (simPiece.segments.size() * 2 > nout) { // note *2 for disjoint + left = mid; + } else { + right = mid; + } + } + // TODO + SimPiece simPiece = new SimPiece(ts.data, left); + int n1 = simPiece.segments.size() * 2; + simPiece = new SimPiece(ts.data, right); + int n2 = simPiece.segments.size() * 2; + if (Math.abs(n1 - nout) < Math.abs(n2 - nout)) { + return left; + } else { + return right; + } + // + } +}
