This is an automated email from the ASF dual-hosted git repository. jiangtian pushed a commit to branch deletion_expr in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit f91f273486e649115bf8b601fd53fe876660770b Author: Tian Jiang <[email protected]> AuthorDate: Thu Sep 19 10:04:47 2024 +0800 add expr framework --- .../org/apache/iotdb/db/expr/DeletionExprMain.java | 301 +++++++++++++++++++++ .../iotdb/db/expr/conf/SimulationConfig.java | 42 +++ .../expr/distribution/FixedIntervalGenerator.java | 35 +++ .../apache/iotdb/db/expr/entity/SimDeletion.java | 30 ++ .../apache/iotdb/db/expr/entity/SimModFile.java | 41 +++ .../iotdb/db/expr/entity/SimModFileManager.java | 27 ++ .../org/apache/iotdb/db/expr/entity/SimTsFile.java | 66 +++++ .../iotdb/db/expr/entity/SimTsFileManager.java | 27 ++ .../iotdb/db/expr/entity/SimpleModAllocator.java | 90 ++++++ .../java/org/apache/iotdb/db/expr/event/Event.java | 54 ++++ .../db/expr/event/ExecuteLastPointQueryEvent.java | 53 ++++ .../db/expr/event/ExecuteRangeQueryEvent.java | 97 +++++++ .../iotdb/db/expr/event/GenerateDeletionEvent.java | 61 +++++ .../iotdb/db/expr/event/GenerateTsFileEvent.java | 70 +++++ .../iotdb/db/expr/executor/EventExecutor.java | 27 ++ .../iotdb/db/expr/executor/SimpleExecutor.java | 113 ++++++++ .../iotdb/db/expr/simulator/SimpleSimulator.java | 126 +++++++++ .../iotdb/db/expr/simulator/SimulationContext.java | 30 ++ .../db/expr/simulator/SimulationStatistics.java | 77 ++++++ 19 files changed, 1367 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/DeletionExprMain.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/DeletionExprMain.java new file mode 100644 index 00000000000..e17ac5f717a --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/DeletionExprMain.java @@ -0,0 +1,301 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.expr; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import org.apache.iotdb.db.expr.conf.SimulationConfig; +import org.apache.iotdb.db.expr.distribution.FixedIntervalGenerator; +import org.apache.iotdb.db.expr.entity.SimDeletion; +import org.apache.iotdb.db.expr.entity.SimTsFile; +import org.apache.iotdb.db.expr.entity.SimpleModAllocator; +import org.apache.iotdb.db.expr.event.Event; +import org.apache.iotdb.db.expr.event.ExecuteLastPointQueryEvent; +import org.apache.iotdb.db.expr.event.ExecuteRangeQueryEvent; +import org.apache.iotdb.db.expr.event.GenerateDeletionEvent; +import org.apache.iotdb.db.expr.event.GenerateTsFileEvent; +import org.apache.iotdb.db.expr.simulator.SimpleSimulator; + +import org.apache.tsfile.read.common.TimeRange; + +public class DeletionExprMain { + + private SimulationConfig config; + private SimpleSimulator simulator; + private SimpleModAllocator simpleModAllocator; + private long maxStep; + private long maxTimestamp; + private ExprReport report; + private static int maxFileCntThreshold = 30; + + public DeletionExprMain() { + init(); + initReport(); + } + + public void init() { + config = new SimulationConfig(); + simulator = new SimpleSimulator(config); + simpleModAllocator = new SimpleModAllocator(config, + simulator.getSimulationContext()); + maxStep = 10000; + maxTimestamp = Long.MAX_VALUE; + } + + public void initReport() { + report = new ExprReport(); + } + + private List<Event> initEvents() { + List<Event> events = new ArrayList<>(); + + SimTsFile initTsFile = + new SimTsFile(0, new TimeRange(0, config.tsfileRange), simpleModAllocator::allocate); + GenerateTsFileEvent generateTsFileEvent = + new GenerateTsFileEvent( + config, + initTsFile, + config.tsfileRange, + new FixedIntervalGenerator(config.generateTsFileInterval)); + events.add(generateTsFileEvent); + + GenerateDeletionEvent generatePartialDeletionEvent = + new GenerateDeletionEvent( + config, + new SimDeletion(new TimeRange(config.partialDeletionOffset, + config.partialDeletionOffset + config.partialDeletionRange)), + config.partialDeletionStep, + new FixedIntervalGenerator(config.generatePartialDeletionInterval)); + generatePartialDeletionEvent.generateTimestamp = config.deletionStartTime; + GenerateDeletionEvent generateFullDeletionEvent = + new GenerateDeletionEvent( + config, + new SimDeletion(new TimeRange(0, Long.MAX_VALUE)), + 0, + new FixedIntervalGenerator(config.generateFullDeletionInterval)); + generateFullDeletionEvent.generateTimestamp = config.deletionStartTime; + events.add(generatePartialDeletionEvent); + events.add(generateFullDeletionEvent); + + ExecuteRangeQueryEvent executeRangeQueryEvent = + new ExecuteRangeQueryEvent( + config, + new TimeRange(config.rangeQueryOffset, + config.rangeQueryRange + config.rangeQueryOffset), + config.rangeQueryStep, + new FixedIntervalGenerator(config.rangeQueryInterval)); + ExecuteLastPointQueryEvent executeLastPointQueryEvent = new ExecuteLastPointQueryEvent(config, + new TimeRange(0, 1), new FixedIntervalGenerator(config.pointQueryInterval)); + ExecuteRangeQueryEvent executeFullQueryEvent = new ExecuteRangeQueryEvent(config, + new TimeRange(Long.MIN_VALUE, Long.MAX_VALUE), 0, + new FixedIntervalGenerator(config.fullQueryInterval)); + events.add(executeRangeQueryEvent); + events.add(executeLastPointQueryEvent); + events.add(executeFullQueryEvent); + + return events; + } + + public void doExpr(boolean printState) { + simulator.maxStep = maxStep; + simulator.maxTimestamp = maxTimestamp; + + List<Event> events = initEvents(); + simulator.addEvents(events); + + simulator.start(); + + writeReport(); + + if (printState) { + System.out.println(simulator); + System.out.println(simulator.getStatistics()); + System.out.println(simulator.getSimulationContext().modFileManager.modFileList.stream() + .map(s -> s.mods.size()).collect( + Collectors.toList())); + System.out.println(simulator.getSimulationContext().modFileManager.modFileList.stream() + .map(s -> s.tsfileReferences.size()).collect( + Collectors.toList())); + } + } + + private void writeReport() { + report.deletionWriteTime = simulator.getStatistics().partialDeletionExecutedTime + + simulator.getStatistics().fullDeletionExecutedTime; + report.deletionTimeList.add(report.deletionWriteTime); + report.deletionReadTime = simulator.getStatistics().queryReadDeletionTime; + report.queryTimeList.add(report.deletionReadTime); + report.totalTimeList.add( + (long) (report.deletionWriteTime * config.writeTimeWeight) + report.deletionReadTime); + } + + public static class ExprReport { + + public List<Long> deletionTimeList = new ArrayList<>(); + public List<Long> queryTimeList = new ArrayList<>(); + public List<Long> totalTimeList = new ArrayList<>(); + public long deletionWriteTime; + public long deletionReadTime; + + public void print() { + System.out.println(deletionTimeList); + System.out.println(queryTimeList); + System.out.println(totalTimeList); + } + } + + @FunctionalInterface + public interface Configurer { + + void configure(DeletionExprMain exprMain, int i); + } + + public static ExprReport oneExpr(Configurer configurer, int exprNum) { + // baseline, each TsFile has one Mod File + DeletionExprMain expr = new DeletionExprMain(); + + initExpr(expr); + configurer.configure(expr, exprNum); + expr.config.modFileCntThreshold = Integer.MAX_VALUE; + expr.config.modFileSizeThreshold = 0; + expr.doExpr(true); + + // use modFileCntThreshold as the x-axis + for (int i = 1; i < expr.maxFileCntThreshold; i++) { + initExpr(expr); + configurer.configure(expr, exprNum); + expr.config.modFileCntThreshold = i; + expr.doExpr(true); + } + + return expr.report; + } + + public static void initExpr(DeletionExprMain expr) { + expr.init(); + + expr.maxStep = Long.MAX_VALUE; + expr.maxTimestamp = 24 * 60 * 60 * 1000 * 1000L; + expr.config.rangeQueryInterval = 1000_000; + expr.config.fullQueryInterval = 1000_000; + expr.config.pointQueryInterval = 1000_000; + expr.config.generateFullDeletionInterval = 2_000_000; + expr.config.generatePartialDeletionInterval = 2_000_000; + expr.config.partialDeletionRange = expr.config.tsfileRange * 3; + expr.config.partialDeletionOffset = -expr.config.partialDeletionRange; + expr.config.partialDeletionStep = (long) (expr.config.tsfileRange / ( + 1.0 * expr.config.generateTsFileInterval / expr.config.generatePartialDeletionInterval)); + + expr.config.generateTsFileInterval = 10_000_000; + expr.config.modFileSizeThreshold = 64 * 1024; + expr.config.deletionStartTime = 1000 * expr.config.generateTsFileInterval; +// expr.config.queryRange = maxTimestamp; +// expr.config.queryStep = 0; + expr.config.rangeQueryRange = expr.config.tsfileRange * 1000; + expr.config.rangeQueryStep = expr.config.tsfileRange / (expr.config.generateTsFileInterval + / expr.config.rangeQueryInterval); + expr.config.rangeQueryOffset = -expr.config.rangeQueryRange + + expr.config.deletionStartTime / expr.config.generateTsFileInterval + * expr.config.tsfileRange; + } + + private static void parallelExpr(Configurer configurer, int exprNum, Function<Integer, String> argsToString) + throws ExecutionException, InterruptedException { + ExecutorService service = Executors.newCachedThreadPool(); + List<Future<ExprReport>> asyncReports = new ArrayList<>(); + for (int i = 0; i < exprNum; i++) { + int finalI = i; + asyncReports.add(service.submit(() -> oneExpr(configurer, + finalI))); + } + + for (Future<ExprReport> asyncReport : asyncReports) { + asyncReport.get(); + } + + for (int i = 0; i < asyncReports.size(); i++) { + System.out.println(argsToString.apply(i)); + asyncReports.get(i).get().print(); + } + service.shutdownNow(); + } + + private static void testSizeThreshold() throws ExecutionException, InterruptedException { + String argName = "sizeThreshold"; + long[] exprArgs = new long[]{ + 16 * 1024, + 32 * 1024, + 64 * 1024, + 128 * 1024, + 256 * 1024, + }; + Configurer configurer = (expr, j) -> { + expr.config.modFileSizeThreshold = exprArgs[j]; + }; + parallelExpr(configurer, exprArgs.length, (i) -> argName + ":" + exprArgs[i]); + } + + private static void testQueryInterval() throws ExecutionException, InterruptedException { + String argName = "queryInterval"; + long[] exprArgs = new long[]{ + 500_000, + 1000_000, + 1500_000, + 2000_000, + 2500_000 + }; + Configurer configurer = (expr, j) -> { + expr.config.pointQueryInterval = exprArgs[j]; + expr.config.rangeQueryInterval = exprArgs[j]; + expr.config.fullQueryInterval = exprArgs[j]; + }; + parallelExpr(configurer, exprArgs.length, (i) -> argName + ":" + exprArgs[i]); + } + + private static void testSimulationTime() throws ExecutionException, InterruptedException { + String argName = "simulationTime"; + long[] exprArgs = new long[]{ +// 24 * 60 * 60 * 1000 * 1000L, +// 2 * 24 * 60 * 60 * 1000 * 1000L, + 3 * 24 * 60 * 60 * 1000 * 1000L, +// 4 * 24 * 60 * 60 * 1000 * 1000L, +// 5 * 24 * 60 * 60 * 1000 * 1000L + }; + Configurer configurer = (expr, j) -> { + expr.maxTimestamp = exprArgs[j]; + }; + parallelExpr(configurer, exprArgs.length, (i) -> argName + ":" + exprArgs[i]); + } + + public static void main(String[] args) throws ExecutionException, InterruptedException { + maxFileCntThreshold = 30; + +// testSizeThreshold(); +// testQueryInterval(); + testSimulationTime(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/conf/SimulationConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/conf/SimulationConfig.java new file mode 100644 index 00000000000..83ea23cc578 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/conf/SimulationConfig.java @@ -0,0 +1,42 @@ +package org.apache.iotdb.db.expr.conf; + +public class SimulationConfig { + + // if the time range is [0, 1000], then the file size is 10_000 bytes + public long timeRangeToBytesFactor = 10L; + // when using us timestamp, 10MB/s = 10B/us + public long IoBandwidthBytesPerTimestamp = 10; + // assume 2ms seek time + public long IoSeekTimestamp = 2_000; + public long deletionSizeInByte = 50; + + public long modFileSizeThreshold = 16 * 1024; + public int modFileCntThreshold = 10; + + public long generateTsFileInterval = 10_000_000L; + public long tsfileRange = 10_000_000L; + + // the first query/deletion occurs after writing 10 files + public long deletionStartTime = tsfileRange * 10; + + public long generatePartialDeletionInterval = 20_000_000L; + public long generateFullDeletionInterval = 20_000_000L; + // the first deletion ranges from [partialDeletionOffset, partialDeletionRange + partialDeletionOffset], + // and the next one ranges from [partialDeletionOffset + partialDeletionStep, partialDeletionRange + partialDeletionOffset + partialDeletionStep], + // and so on + public long partialDeletionRange = tsfileRange * 3; + public long partialDeletionStep = tsfileRange / 2; + public long partialDeletionOffset = -partialDeletionRange; + + public long rangeQueryInterval = 50_000; + public long rangeQueryRange = tsfileRange * 10; + public long rangeQueryOffset = rangeQueryRange; + // if a tsFile is generated every X queries, then the step should be 1/X of the file range, + // so that the query end time will not exceed data time + public long rangeQueryStep = tsfileRange / (generateTsFileInterval / rangeQueryInterval); + + public long fullQueryInterval = 50_000; + public long pointQueryInterval = 50_000; + + public double writeTimeWeight = 100.0; +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/distribution/FixedIntervalGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/distribution/FixedIntervalGenerator.java new file mode 100644 index 00000000000..5d6c022fe42 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/distribution/FixedIntervalGenerator.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.expr.distribution; + +import java.util.function.Supplier; + +public class FixedIntervalGenerator implements Supplier<Long> { + private long interval; + + public FixedIntervalGenerator(long interval) { + this.interval = interval; + } + + @Override + public Long get() { + return interval; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/entity/SimDeletion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/entity/SimDeletion.java new file mode 100644 index 00000000000..e90d92ab11e --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/entity/SimDeletion.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.expr.entity; + +import org.apache.tsfile.read.common.TimeRange; + +public class SimDeletion { + public TimeRange timeRange; + + public SimDeletion(TimeRange timeRange) { + this.timeRange = timeRange; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/entity/SimModFile.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/entity/SimModFile.java new file mode 100644 index 00000000000..e16bde533aa --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/entity/SimModFile.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.expr.entity; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +public class SimModFile { + public Set<SimTsFile> tsfileReferences = new HashSet<>(); + public List<SimDeletion> mods = new ArrayList<>(); + public int fullDeletionCnt; + public int partialDeletionCnt; + + public void add(SimDeletion deletion) { + mods.add(deletion); + if (deletion.timeRange.getMax() == Long.MAX_VALUE) { + fullDeletionCnt++; + } else { + partialDeletionCnt++; + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/entity/SimModFileManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/entity/SimModFileManager.java new file mode 100644 index 00000000000..51c37141dea --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/entity/SimModFileManager.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.expr.entity; + +import java.util.ArrayList; +import java.util.List; + +public class SimModFileManager { + public List<SimModFile> modFileList = new ArrayList<>(); +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/entity/SimTsFile.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/entity/SimTsFile.java new file mode 100644 index 00000000000..27d9106a439 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/entity/SimTsFile.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.expr.entity; + +import org.apache.tsfile.read.common.TimeRange; + +import java.util.Collections; +import java.util.List; +import java.util.function.Function; + +public class SimTsFile { + public long version; + public TimeRange timeRange; + + private SimModFile modFile; + public final Function<SimTsFile, SimModFile> modFileAllocator; + public int startPosInMod; + + public SimTsFile( + long version, TimeRange timeRange, Function<SimTsFile, SimModFile> modFileAllocator) { + this.version = version; + this.timeRange = timeRange; + this.modFileAllocator = modFileAllocator; + } + + public boolean shouldDelete(SimDeletion deletion) { + return deletion.timeRange.overlaps(timeRange); + } + + public SimModFile getModFileMayAllocate() { + if (modFile == null) { + modFile = modFileAllocator.apply(this); + modFile.tsfileReferences.add(this); + startPosInMod = modFile.mods.size(); + } + return modFile; + } + + public List<SimDeletion> getDeletions() { + if (modFile == null) { + return Collections.emptyList(); + } + return modFile.mods.subList(startPosInMod, modFile.mods.size()); + } + + public boolean hasModFile() { + return modFile != null; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/entity/SimTsFileManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/entity/SimTsFileManager.java new file mode 100644 index 00000000000..190ac0d13d2 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/entity/SimTsFileManager.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.expr.entity; + +import java.util.ArrayList; +import java.util.List; + +public class SimTsFileManager { + public List<SimTsFile> tsFileList = new ArrayList<>(); +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/entity/SimpleModAllocator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/entity/SimpleModAllocator.java new file mode 100644 index 00000000000..7825140d156 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/entity/SimpleModAllocator.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.expr.entity; + +import org.apache.iotdb.db.expr.conf.SimulationConfig; +import org.apache.iotdb.db.expr.simulator.SimpleSimulator.SimpleContext; + +public class SimpleModAllocator { + + private SimulationConfig config; + private SimpleContext context; + + public SimpleModAllocator(SimulationConfig config, SimpleContext context) { + this.config = config; + this.context = context; + } + + private SimModFile tryAllocate(SimTsFile tsFile) { + if (tsFile.hasModFile()) { + SimModFile prevModFile = tsFile.getModFileMayAllocate(); + if (context.modFileManager.modFileList.size() < config.modFileCntThreshold) { + // can allocate more mod file + long totalSize = prevModFile.mods.size() * config.deletionSizeInByte; + if (totalSize > config.modFileSizeThreshold) { +// System.out.printf( +// "When allocating new Mod File, there are %d partial deletion and %d full deletion%n", +// prevModFile.partialDeletionCnt, prevModFile.fullDeletionCnt); + // the previous one is already large enough, allocate a new one + return allocateNew(); + } + } + // share the previous one + return prevModFile; + } + return null; + } + + private SimModFile allocateNew() { + SimModFile newModFile = new SimModFile(); + context.modFileManager.modFileList.add(newModFile); + context.getStatistics().modFileGeneratedCnt++; + return newModFile; + } + + public SimModFile allocate(SimTsFile tsFile) { + int filePos = context.tsFileManager.tsFileList.indexOf(tsFile); + + int forwardIndex = filePos + 1; + int backwardIndex = filePos - 1; + while (forwardIndex < context.tsFileManager.tsFileList.size() || backwardIndex >= 0) { + if (forwardIndex < context.tsFileManager.tsFileList.size()) { + SimTsFile nextFile = context.tsFileManager.tsFileList.get(forwardIndex); + SimModFile simModFile = tryAllocate(nextFile); + if (simModFile != null) { + return simModFile; + } + forwardIndex++; + } + + if (backwardIndex >= 0) { + SimTsFile prevFile = context.tsFileManager.tsFileList.get(backwardIndex); + SimModFile simModFile = tryAllocate(prevFile); + if (simModFile != null) { + return simModFile; + } + backwardIndex--; + } + } + + // no mod File + return allocateNew(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/event/Event.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/event/Event.java new file mode 100644 index 00000000000..61a5361e4ea --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/event/Event.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.expr.event; + +import org.apache.iotdb.db.expr.conf.SimulationConfig; +import org.apache.iotdb.db.expr.simulator.SimulationContext; + +import java.util.List; + +public abstract class Event implements Comparable<Event> { + public long generateTimestamp; + public long executionTimestamp; + public long executionStep; + + protected SimulationConfig config; + protected long timeConsumption = -1; + + public Event(SimulationConfig config) { + this.config = config; + } + + public abstract List<Event> nextEvents(SimulationContext context); + + @Override + public int compareTo(Event o) { + return Long.compare(this.generateTimestamp, o.generateTimestamp); + } + + public long getTimeConsumption() { + if (timeConsumption == -1) { + timeConsumption = calTimeConsumption(); + } + return timeConsumption; + } + + protected abstract long calTimeConsumption(); +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/event/ExecuteLastPointQueryEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/event/ExecuteLastPointQueryEvent.java new file mode 100644 index 00000000000..99853004d70 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/event/ExecuteLastPointQueryEvent.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.expr.event; + +import java.util.Collections; +import java.util.List; +import java.util.function.Supplier; +import org.apache.iotdb.db.expr.conf.SimulationConfig; +import org.apache.iotdb.db.expr.entity.SimDeletion; +import org.apache.iotdb.db.expr.entity.SimTsFile; +import org.apache.iotdb.db.expr.simulator.SimulationContext; +import org.apache.tsfile.read.common.TimeRange; + +public class ExecuteLastPointQueryEvent extends ExecuteRangeQueryEvent { + + public ExecuteLastPointQueryEvent( + SimulationConfig config, TimeRange timeRange, Supplier<Long> intervalGenerator) { + super(config, timeRange, 0, intervalGenerator); + } + + protected TimeRange nextTimeRange(SimulationContext context) { + long currentTimestamp = context.getSimulator().currentTimestamp; + long lastTsFileVersion = currentTimestamp / context.getConfig().generateTsFileInterval; + long lastTsFileTime = lastTsFileVersion * context.getConfig().tsfileRange; + return new TimeRange(lastTsFileTime,lastTsFileTime + 1); + } + + @Override + public List<Event> nextEvents(SimulationContext context) { + ExecuteLastPointQueryEvent event = + new ExecuteLastPointQueryEvent(config, nextTimeRange(context), intervalGenerator); + event.generateTimestamp = + context.getSimulator().getCurrentTimestamp() + intervalGenerator.get(); + return Collections.singletonList(event); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/event/ExecuteRangeQueryEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/event/ExecuteRangeQueryEvent.java new file mode 100644 index 00000000000..0e547e7ccb5 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/event/ExecuteRangeQueryEvent.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.expr.event; + +import org.apache.iotdb.db.expr.conf.SimulationConfig; +import org.apache.iotdb.db.expr.entity.SimDeletion; +import org.apache.iotdb.db.expr.entity.SimTsFile; +import org.apache.iotdb.db.expr.simulator.SimulationContext; + +import org.apache.tsfile.read.common.TimeRange; + +import java.util.Collections; +import java.util.List; +import java.util.function.Supplier; + +public class ExecuteRangeQueryEvent extends Event { + + public TimeRange timeRange; + public long step; + protected final Supplier<Long> intervalGenerator; + + public List<SimTsFile> queriedTsFiles; + public List<List<SimDeletion>> queriedDeletions; + + public double readTsFileTimeSum = 0; + public double readDeletionTimeSum = 0; + public double readDeletionSeekTimeSum = 0; + public double readDeletionTransTimeSum = 0; + + public ExecuteRangeQueryEvent( + SimulationConfig config, TimeRange timeRange, long step, Supplier<Long> intervalGenerator) { + super(config); + this.timeRange = timeRange; + this.step = step; + this.intervalGenerator = intervalGenerator; + } + + protected TimeRange nextTimeRange(SimulationContext context) { + return new TimeRange(timeRange.getMin() + step, timeRange.getMax() + step); + } + + @Override + public List<Event> nextEvents(SimulationContext context) { + ExecuteRangeQueryEvent event = + new ExecuteRangeQueryEvent(config, nextTimeRange(context), step, intervalGenerator); + event.generateTimestamp = + context.getSimulator().getCurrentTimestamp() + intervalGenerator.get(); + return Collections.singletonList(event); + } + + protected double readTsFileConsumption(SimTsFile tsFile) { + return 1.0 + * (tsFile.timeRange.getMax() - tsFile.timeRange.getMin()) + * config.timeRangeToBytesFactor + / config.IoBandwidthBytesPerTimestamp + + config.IoSeekTimestamp; + } + + protected double readDeletionsConsumption(List<SimDeletion> deletions) { + double transTime = + 1.0 * deletions.size() * config.deletionSizeInByte / config.IoBandwidthBytesPerTimestamp; + readDeletionTransTimeSum += transTime; + readDeletionSeekTimeSum += config.IoSeekTimestamp; + return transTime + + config.IoSeekTimestamp; + } + + @Override + public long calTimeConsumption() { + for (int i = 0; i < queriedTsFiles.size(); i++) { + SimTsFile simTsFile = queriedTsFiles.get(i); + readTsFileTimeSum += readTsFileConsumption(simTsFile); + if (simTsFile.hasModFile()) { + readDeletionTimeSum += readDeletionsConsumption(queriedDeletions.get(i)); + } + } + + return Math.round(readTsFileTimeSum + readDeletionTimeSum); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/event/GenerateDeletionEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/event/GenerateDeletionEvent.java new file mode 100644 index 00000000000..4b1ac0b1767 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/event/GenerateDeletionEvent.java @@ -0,0 +1,61 @@ +package org.apache.iotdb.db.expr.event; + +import org.apache.iotdb.db.expr.conf.SimulationConfig; +import org.apache.iotdb.db.expr.entity.SimDeletion; +import org.apache.iotdb.db.expr.entity.SimModFile; +import org.apache.iotdb.db.expr.simulator.SimulationContext; + +import org.apache.tsfile.read.common.TimeRange; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.function.Supplier; + +public class GenerateDeletionEvent extends Event { + + public SimDeletion currentDeletion; + public long step; + private final Supplier<Long> intervalGenerator; + + public Collection<SimModFile> involvedModFiles; + + public GenerateDeletionEvent( + SimulationConfig config, + SimDeletion currentDeletion, + long step, + Supplier<Long> intervalGenerator) { + super(config); + + this.currentDeletion = currentDeletion; + this.step = step; + this.intervalGenerator = intervalGenerator; + } + + public SimDeletion nextDeletion() { + return new SimDeletion( + new TimeRange( + currentDeletion.timeRange.getMin() + step, currentDeletion.timeRange.getMax() + step)); + } + + @Override + public List<Event> nextEvents(SimulationContext context) { + GenerateDeletionEvent event = + new GenerateDeletionEvent(config, nextDeletion(), step, intervalGenerator); + event.generateTimestamp = + context.getSimulator().getCurrentTimestamp() + intervalGenerator.get(); + return Collections.singletonList(event); + } + + @Override + public long calTimeConsumption() { + double sum = 0.0; + sum += + involvedModFiles.size() * config.IoSeekTimestamp + + 1.0 + * involvedModFiles.size() + * config.deletionSizeInByte + / config.IoBandwidthBytesPerTimestamp; + return Math.round(sum); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/event/GenerateTsFileEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/event/GenerateTsFileEvent.java new file mode 100644 index 00000000000..8525744cc17 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/event/GenerateTsFileEvent.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.expr.event; + +import org.apache.iotdb.db.expr.conf.SimulationConfig; +import org.apache.iotdb.db.expr.entity.SimTsFile; +import org.apache.iotdb.db.expr.simulator.SimulationContext; + +import org.apache.tsfile.read.common.TimeRange; + +import java.util.Collections; +import java.util.List; +import java.util.function.Supplier; + +public class GenerateTsFileEvent extends Event { + + public SimTsFile currentTSFile; + public long step; + private final Supplier<Long> intervalGenerator; + + public GenerateTsFileEvent( + SimulationConfig config, + SimTsFile currentTSFile, + long step, + Supplier<Long> intervalGenerator) { + super(config); + this.currentTSFile = currentTSFile; + this.step = step; + this.intervalGenerator = intervalGenerator; + } + + public SimTsFile nextTSFile() { + return new SimTsFile( + currentTSFile.version + 1, + new TimeRange( + currentTSFile.timeRange.getMin() + step, currentTSFile.timeRange.getMax() + step), + currentTSFile.modFileAllocator); + } + + @Override + public List<Event> nextEvents(SimulationContext context) { + GenerateTsFileEvent event = + new GenerateTsFileEvent(config, nextTSFile(), step, intervalGenerator); + event.generateTimestamp = + context.getSimulator().getCurrentTimestamp() + intervalGenerator.get(); + return Collections.singletonList(event); + } + + @Override + public long calTimeConsumption() { + return 0; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/executor/EventExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/executor/EventExecutor.java new file mode 100644 index 00000000000..4c3c57cd8aa --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/executor/EventExecutor.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.expr.executor; + +import org.apache.iotdb.db.expr.event.Event; +import org.apache.iotdb.db.expr.simulator.SimulationContext; + +public interface EventExecutor<T extends SimulationContext> { + void execute(Event event, T context); +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/executor/SimpleExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/executor/SimpleExecutor.java new file mode 100644 index 00000000000..50c826cb216 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/executor/SimpleExecutor.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.expr.executor; + +import org.apache.iotdb.db.expr.entity.SimDeletion; +import org.apache.iotdb.db.expr.entity.SimModFile; +import org.apache.iotdb.db.expr.entity.SimTsFile; +import org.apache.iotdb.db.expr.event.Event; +import org.apache.iotdb.db.expr.event.ExecuteRangeQueryEvent; +import org.apache.iotdb.db.expr.event.GenerateDeletionEvent; +import org.apache.iotdb.db.expr.event.GenerateTsFileEvent; +import org.apache.iotdb.db.expr.simulator.SimpleSimulator.SimpleContext; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +public class SimpleExecutor implements EventExecutor<SimpleContext> { + + @Override + public void execute(Event event, SimpleContext context) { + if (event instanceof GenerateTsFileEvent) { + doExecute(((GenerateTsFileEvent) event), context); + } else if (event instanceof GenerateDeletionEvent) { + doExecute(((GenerateDeletionEvent) event), context); + } else if (event instanceof ExecuteRangeQueryEvent) { + doExecute(((ExecuteRangeQueryEvent) event), context); + } + + event.executionStep = context.getSimulator().getCurrentStep(); + event.executionTimestamp = context.getSimulator().getCurrentTimestamp(); + } + + private void doExecute(GenerateTsFileEvent event, SimpleContext simpleContext) { + simpleContext.tsFileManager.tsFileList.add(event.currentTSFile); + + simpleContext.getStatistics().tsFileGeneratedCnt++; + } + + private void doExecute(GenerateDeletionEvent event, SimpleContext simpleContext) { + Set<SimModFile> involvedModFiles = new HashSet<>(); + for (SimTsFile simTsFile : simpleContext.tsFileManager.tsFileList) { + if (simTsFile.shouldDelete(event.currentDeletion)) { + involvedModFiles.add(simTsFile.getModFileMayAllocate()); + } + } + + for (SimModFile involvedModFile : involvedModFiles) { + involvedModFile.add(event.currentDeletion); + } + event.involvedModFiles = involvedModFiles; + + simpleContext.getStatistics().totalDeletionWriteBytes += + involvedModFiles.size() * simpleContext.getConfig().deletionSizeInByte; + if (event.currentDeletion.timeRange.getMax() != Long.MAX_VALUE) { + simpleContext.getStatistics().partialDeletionExecutedCnt++; + simpleContext.getStatistics().partialDeletionExecutedTime += event.getTimeConsumption(); + } else { + simpleContext.getStatistics().fullDeletionExecutedCnt++; + simpleContext.getStatistics().fullDeletionExecutedTime += event.getTimeConsumption(); + } + } + + private void doExecute(ExecuteRangeQueryEvent event, SimpleContext simpleContext) { + List<SimTsFile> queriedTsFiles = new ArrayList<>(); + List<List<SimDeletion>> queriedDeletions = new ArrayList<>(); + for (SimTsFile simTsFile : simpleContext.tsFileManager.tsFileList) { + if (simTsFile.timeRange.overlaps(event.timeRange)) { + queriedTsFiles.add(simTsFile); + List<SimDeletion> deletions = simTsFile.getDeletions(); + queriedDeletions.add(deletions); + if (!deletions.isEmpty()) { + simpleContext.getStatistics().totalDeletionReadBytes += + deletions.size() * simpleContext.getConfig().deletionSizeInByte; + simpleContext.getStatistics().queryReadDeletionCnt += deletions.size(); + simpleContext.getStatistics().queryReadModsCnt += 1; + } + } + } + + event.queriedTsFiles = queriedTsFiles; + event.queriedDeletions = queriedDeletions; + + simpleContext.getStatistics().queryExecutedCnt++; + simpleContext.getStatistics().queryExecutedTime += event.getTimeConsumption(); + simpleContext.getStatistics().queryReadDeletionTime += Math.round(event.readDeletionTimeSum); + simpleContext.getStatistics().queryReadDeletionSeekTime += Math.round( + event.readDeletionSeekTimeSum); + simpleContext.getStatistics().queryReadDeletionTransTime += Math.round( + event.readDeletionTransTimeSum); +// System.out.println( +// simpleContext.getSimulator().currentStep + " " + simpleContext.getSimulator().currentTimestamp +// + " " + event.readDeletionSeekTimeSum + " " + event.readDeletionTransTimeSum); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/simulator/SimpleSimulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/simulator/SimpleSimulator.java new file mode 100644 index 00000000000..58aacbfcdb9 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/simulator/SimpleSimulator.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.expr.simulator; + +import java.util.List; +import org.apache.iotdb.db.expr.conf.SimulationConfig; +import org.apache.iotdb.db.expr.entity.SimModFileManager; +import org.apache.iotdb.db.expr.entity.SimTsFileManager; +import org.apache.iotdb.db.expr.event.Event; +import org.apache.iotdb.db.expr.executor.SimpleExecutor; + +import java.util.PriorityQueue; + +public class SimpleSimulator { + public long currentStep; + public long currentTimestamp; + private final PriorityQueue<Event> eventQueue = new PriorityQueue<>(); + private final SimpleExecutor eventExecutor = new SimpleExecutor(); + private final SimpleContext simulationContext = new SimpleContext(); + private final SimulationStatistics statistics = new SimulationStatistics(); + private final SimulationConfig simulationConfig; + + public long maxStep = Long.MAX_VALUE; + public long maxTimestamp = Long.MAX_VALUE; + + public SimpleSimulator(SimulationConfig config) { + this.simulationConfig = config; + } + + public void addEvent(Event event) { + eventQueue.add(event); + } + + public void addEvents(List<Event> events) { + eventQueue.addAll(events); + } + + public void start() { + doSimulate(); + } + + public void forwardTimeTo(long nextTimestamp) { + currentTimestamp = nextTimestamp; + } + + public void forwardTime(long delta) { + currentTimestamp += delta; + } + + private void doSimulate() { + while (!eventQueue.isEmpty() && currentTimestamp < maxTimestamp && currentStep < maxStep) { + Event event = eventQueue.poll(); + if (event.generateTimestamp > currentTimestamp) { + forwardTimeTo(event.generateTimestamp); + } + eventExecutor.execute(event, simulationContext); + addEvents(event.nextEvents(simulationContext)); + currentStep++; + } + } + + public long getCurrentStep() { + return currentStep; + } + + public long getCurrentTimestamp() { + return currentTimestamp; + } + + public SimulationStatistics getStatistics() { + return statistics; + } + + public SimpleContext getSimulationContext() { + return simulationContext; + } + + @Override + public String toString() { + return "SimpleSimulator{" + + "currentTimestamp=" + currentTimestamp + + "\n, currentStep=" + currentStep + + "\n, eventQueue=" + eventQueue + + "\n, maxStep=" + maxStep + + "\n, maxTimestamp=" + maxTimestamp + + '}'; + } + + public class SimpleContext implements SimulationContext { + + public SimTsFileManager tsFileManager = new SimTsFileManager(); + public SimModFileManager modFileManager = new SimModFileManager(); + + @Override + public SimpleSimulator getSimulator() { + return SimpleSimulator.this; + } + + @Override + public SimulationStatistics getStatistics() { + return statistics; + } + + @Override + public SimulationConfig getConfig() { + return simulationConfig; + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/simulator/SimulationContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/simulator/SimulationContext.java new file mode 100644 index 00000000000..048e17c0971 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/simulator/SimulationContext.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.expr.simulator; + +import org.apache.iotdb.db.expr.conf.SimulationConfig; + +public interface SimulationContext { + SimpleSimulator getSimulator(); + + SimulationStatistics getStatistics(); + + SimulationConfig getConfig(); +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/simulator/SimulationStatistics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/simulator/SimulationStatistics.java new file mode 100644 index 00000000000..1d24fda6a5f --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/expr/simulator/SimulationStatistics.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.expr.simulator; + +public class SimulationStatistics { + public long tsFileGeneratedCnt; + public long modFileGeneratedCnt; + + public long partialDeletionExecutedCnt; + public long fullDeletionExecutedCnt; + public long partialDeletionExecutedTime; + public long fullDeletionExecutedTime; + + public long queryExecutedCnt; + public long queryReadDeletionTime; + public long queryReadDeletionSeekTime; + public long queryReadDeletionTransTime; + public long queryReadDeletionCnt; + public long queryReadModsCnt; + public long queryExecutedTime; + + public long totalDeletionWriteBytes; + public long totalDeletionReadBytes; + + @Override + public String toString() { + return "SimulationStatistics{" + + "tsFileGeneratedCnt=" + + tsFileGeneratedCnt + + "\n, modFileGeneratedCnt=" + + modFileGeneratedCnt + + "\n, partialDeletionExecutedCnt=" + + partialDeletionExecutedCnt + + "\n, fullDeletionExecutedCnt=" + + fullDeletionExecutedCnt + + "\n, partialDeletionExecutedTime=" + + partialDeletionExecutedTime + + "\n, fullDeletionExecutedTime=" + + fullDeletionExecutedTime + + "\n, queryExecutedCnt=" + + queryExecutedCnt + + "\n, queryReadDeletionTime=" + + queryReadDeletionTime + + "\n, queryReadDeletionSeekTime=" + + queryReadDeletionSeekTime + + "\n, queryReadDeletionTransTime=" + + queryReadDeletionTransTime + + "\n, queryReadDeletionCnt=" + + queryReadDeletionCnt + + "\n, queryReadModsCnt=" + + queryReadModsCnt + + "\n, queryExecutedTime=" + + queryExecutedTime + + "\n, totalDeletionWriteBytes=" + + totalDeletionWriteBytes + + "\n, totalDeletionReadBytes=" + + totalDeletionReadBytes + + '}'; + } +}
