This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch TimeSeriesMetadataCache in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit d5b63e92052a4eeea7882ceda1b53306ccead104 Author: JackieTien97 <[email protected]> AuthorDate: Mon Nov 2 14:56:33 2020 +0800 test --- .../main/java/org/apache/iotdb/SessionExample.java | 157 +++++++++++++++------ .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 7 +- .../iotdb/db/engine/cache/LRULinkedHashMap.java | 2 +- 3 files changed, 114 insertions(+), 52 deletions(-) diff --git a/example/session/src/main/java/org/apache/iotdb/SessionExample.java b/example/session/src/main/java/org/apache/iotdb/SessionExample.java index dc68fe2..8ee9970 100644 --- a/example/session/src/main/java/org/apache/iotdb/SessionExample.java +++ b/example/session/src/main/java/org/apache/iotdb/SessionExample.java @@ -23,10 +23,10 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Random; +import java.util.concurrent.CountDownLatch; import org.apache.iotdb.rpc.BatchExecutionException; import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; -import org.apache.iotdb.rpc.TSStatusCode; import org.apache.iotdb.session.Session; import org.apache.iotdb.session.SessionDataSet; import org.apache.iotdb.session.SessionDataSet.DataIterator; @@ -47,29 +47,29 @@ public class SessionExample { public static void main(String[] args) - throws IoTDBConnectionException, StatementExecutionException { - session = new Session("127.0.0.1", 6667, "root", "root"); + throws IoTDBConnectionException, StatementExecutionException, InterruptedException { + session = new Session("127.0.0.1", 6667, "root", "root", 1, null); session.open(false); - try { - session.setStorageGroup("root.sg1"); - } catch (StatementExecutionException e) { - if (e.getStatusCode() != TSStatusCode.PATH_ALREADY_EXIST_ERROR.getStatusCode()) - throw e; - } - - createTimeseries(); - createMultiTimeseries(); - insertRecord(); - insertTablet(); - insertTablets(); - insertRecords(); - nonQuery(); +// try { +// session.setStorageGroup("root.sg1"); +// } catch (StatementExecutionException e) { +// if (e.getStatusCode() != TSStatusCode.PATH_ALREADY_EXIST_ERROR.getStatusCode()) +// throw e; +// } + +// createTimeseries(); +// createMultiTimeseries(); +// insertRecord(); +// insertTablet(); +// insertTablets(); +// insertRecords(); +// nonQuery(); query(); - rawDataQuery(); - queryByIterator(); - deleteData(); - deleteTimeseries(); +// rawDataQuery(); +// queryByIterator(); +// deleteData(); +// deleteTimeseries(); session.close(); } @@ -142,22 +142,23 @@ public class SessionExample { } private static void insertRecord() throws IoTDBConnectionException, StatementExecutionException { - String deviceId = ROOT_SG1_D1; + List<String> measurements = new ArrayList<>(); List<TSDataType> types = new ArrayList<>(); - measurements.add("s1"); - measurements.add("s2"); - measurements.add("s3"); - types.add(TSDataType.INT64); - types.add(TSDataType.INT64); - types.add(TSDataType.INT64); + for (int i = 0; i < 1000; i++) { + measurements.add("s" + i); + types.add(TSDataType.INT64); + } - for (long time = 0; time < 100; time++) { + for (long time = 0; time < 10; time++) { List<Object> values = new ArrayList<>(); - values.add(1L); - values.add(2L); - values.add(3L); - session.insertRecord(deviceId, time, measurements, types, values); + for (int i = 0; i < 1000; i++) { + values.add(time); + } + for (int i = 0; i < 10; i++) { + session.insertRecord("root.sg1.d" + i, time, measurements, types, values); + } + session.executeNonQueryStatement("flush"); } } @@ -304,13 +305,13 @@ public class SessionExample { // The schema of measurements of one device // only measurementId and data type in MeasurementSchema take effects in Tablet List<MeasurementSchema> schemaList = new ArrayList<>(); - schemaList.add(new MeasurementSchema("s1", TSDataType.INT64)); - schemaList.add(new MeasurementSchema("s2", TSDataType.INT64)); - schemaList.add(new MeasurementSchema("s3", TSDataType.INT64)); + for (int i = 1; i <= 1000; i++) { + schemaList.add(new MeasurementSchema("s" + i, TSDataType.INT64)); + } - Tablet tablet1 = new Tablet(ROOT_SG1_D1, schemaList, 100); - Tablet tablet2 = new Tablet("root.sg1.d2", schemaList, 100); - Tablet tablet3 = new Tablet("root.sg1.d3", schemaList, 100); + Tablet tablet1 = new Tablet("root.sg1.d1", schemaList, 1); + Tablet tablet2 = new Tablet("root.sg1.d2", schemaList, 1); + Tablet tablet3 = new Tablet("root.sg1.d3", schemaList, 1); Map<String, Tablet> tabletMap = new HashMap<>(); tabletMap.put(ROOT_SG1_D1, tablet1); @@ -403,16 +404,78 @@ public class SessionExample { session.deleteTimeseries(paths); } - private static void query() throws IoTDBConnectionException, StatementExecutionException { - SessionDataSet dataSet; - dataSet = session.executeQueryStatement("select * from root.sg1.d1"); - System.out.println(dataSet.getColumnNames()); - dataSet.setFetchSize(1024); // default is 10000 - while (dataSet.hasNext()) { - System.out.println(dataSet.next()); + private static void query() throws InterruptedException { + long startTime = System.nanoTime(); + CountDownLatch countDownLatch = new CountDownLatch(1); + for (int device = 0; device < 1; device++) { + new Thread(new SumTask(device, countDownLatch)).start(); +// for (int i = 0; i < 5; i++) { +// new Thread(new GroupByTask(device, i, i + 1, countDownLatch)).start(); +// } } + countDownLatch.await(); + System.out.println("cost: " + (System.nanoTime() - startTime) / 1_000_000); + } - dataSet.closeOperationHandle(); + private static class GroupByTask implements Runnable { + + private final int device; + private final int start; + private final int end; + private final CountDownLatch countDownLatch; + + public GroupByTask(int device, int start, int end, CountDownLatch countDownLatch) { + this.device = device; + this.start = start; + this.end = end; + this.countDownLatch = countDownLatch; + } + + @Override + public void run() { + SessionDataSet dataSet; + try { + dataSet = session.executeQueryStatement(String.format("select last_value(*) from root.sg1.d%d group by ([%d,%d),1ms)", device, start, end)); + dataSet.setFetchSize(1); // default is 10000 + while (dataSet.hasNext()) { + dataSet.next(); + } + dataSet.closeOperationHandle(); +// System.out.println("Device" + device + " finished " + start + "-group by task!"); + countDownLatch.countDown(); + } catch (StatementExecutionException | IoTDBConnectionException e) { + e.printStackTrace(); + } + } + } + + private static class SumTask implements Runnable { + + private final int device; + private final CountDownLatch countDownLatch; + + + public SumTask(int device, CountDownLatch countDownLatch) { + this.device = device; + this.countDownLatch = countDownLatch; + } + + @Override + public void run() { + SessionDataSet dataSet; + try { + dataSet = session.executeQueryStatement("select sum(*) from root.sg1.d" + device); + dataSet.setFetchSize(1); // default is 10000 + while (dataSet.hasNext()) { + dataSet.next(); + } + dataSet.closeOperationHandle(); +// System.out.println("Device" + device + " finished sum task!"); + countDownLatch.countDown(); + } catch (StatementExecutionException | IoTDBConnectionException e) { + e.printStackTrace(); + } + } } private static void rawDataQuery() throws IoTDBConnectionException, StatementExecutionException { diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index c6c0fb2..64da9c7 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -21,7 +21,6 @@ package org.apache.iotdb.db.conf; import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.PATH_SEPARATOR; import java.io.File; -import java.time.ZoneId; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.iotdb.db.conf.directories.DirectoryManager; @@ -135,7 +134,7 @@ public class IoTDBConfig { */ private long allocateMemoryForReadWithoutCache = Runtime.getRuntime().maxMemory() * 9 / 100; - private volatile int maxQueryDeduplicatedPathNum = 1000; + private volatile int maxQueryDeduplicatedPathNum = 2000; /** * Is dynamic parameter adapter enable. @@ -298,12 +297,12 @@ public class IoTDBConfig { /** * whether to cache meta data(ChunkMetaData and TsFileMetaData) or not. */ - private boolean metaDataCacheEnable = true; + private boolean metaDataCacheEnable = false; /** * Memory allocated for timeSeriesMetaData cache in read process */ - private long allocateMemoryForTimeSeriesMetaDataCache = allocateMemoryForRead / 10; + private long allocateMemoryForTimeSeriesMetaDataCache = allocateMemoryForRead / 150; /** * Memory allocated for chunkMetaData cache in read process diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/LRULinkedHashMap.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/LRULinkedHashMap.java index 5fb0ae9..4b5943c 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/cache/LRULinkedHashMap.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/LRULinkedHashMap.java @@ -55,7 +55,7 @@ public abstract class LRULinkedHashMap<K extends Accountable, V> { protected long averageSize = 0; public LRULinkedHashMap(long maxMemory) { - this.linkedHashMap = new LinkedHashMap<>(INITIAL_CAPACITY, LOAD_FACTOR_MAP); + this.linkedHashMap = new LinkedHashMap<>(INITIAL_CAPACITY, LOAD_FACTOR_MAP, true); this.maxMemory = maxMemory; this.retainMemory = (long) (maxMemory * RETAIN_PERCENT); }
