This is an automated email from the ASF dual-hosted git repository.
ejttianyu pushed a commit to branch dynamic_compaction
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/dynamic_compaction by this
push:
new 35f27e8 add map hitter
35f27e8 is described below
commit 35f27e83f86c6f778b0496ccebcc928409d642cf
Author: EJTTianyu <[email protected]>
AuthorDate: Tue Apr 27 15:43:13 2021 +0800
add map hitter
---
.../db/engine/heavyhitter/QueryHitterManager.java | 3 +
.../db/engine/heavyhitter/QueryHitterStrategy.java | 4 +-
.../engine/heavyhitter/hitter/HashMapHitter.java | 79 ++++++++++++++++++++++
.../iotdb/db/qp/strategy/PhysicalGenerator.java | 6 ++
4 files changed, 91 insertions(+), 1 deletion(-)
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/heavyhitter/QueryHitterManager.java
b/server/src/main/java/org/apache/iotdb/db/engine/heavyhitter/QueryHitterManager.java
index 1c9f982..cff5504 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/heavyhitter/QueryHitterManager.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/heavyhitter/QueryHitterManager.java
@@ -21,6 +21,7 @@ package org.apache.iotdb.db.engine.heavyhitter;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.heavyhitter.hitter.DefaultHitter;
+import org.apache.iotdb.db.engine.heavyhitter.hitter.HashMapHitter;
public class QueryHitterManager {
@@ -32,6 +33,8 @@ public class QueryHitterManager {
private static QueryHeavyHitters loadQueryHitters() {
switch
(IoTDBDescriptor.getInstance().getConfig().getQueryHitterStrategy()) {
+ case HASH_STRATEGY:
+ return new
HashMapHitter(IoTDBDescriptor.getInstance().getConfig().getMaxHitterNum());
case DEFAULT_STRATEGY:
default:
return new
DefaultHitter(IoTDBDescriptor.getInstance().getConfig().getMaxHitterNum());
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/heavyhitter/QueryHitterStrategy.java
b/server/src/main/java/org/apache/iotdb/db/engine/heavyhitter/QueryHitterStrategy.java
index 80a2daf..115f95a 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/heavyhitter/QueryHitterStrategy.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/heavyhitter/QueryHitterStrategy.java
@@ -21,6 +21,8 @@ package org.apache.iotdb.db.engine.heavyhitter;
public enum QueryHitterStrategy {
//用于测试的 strategy
- DEFAULT_STRATEGY;
+ DEFAULT_STRATEGY,
+ //Hash Map Hitter;
+ HASH_STRATEGY
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/heavyhitter/hitter/HashMapHitter.java
b/server/src/main/java/org/apache/iotdb/db/engine/heavyhitter/hitter/HashMapHitter.java
new file mode 100644
index 0000000..5a91728
--- /dev/null
+++
b/server/src/main/java/org/apache/iotdb/db/engine/heavyhitter/hitter/HashMapHitter.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.heavyhitter.hitter;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.OutputStreamWriter;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.iotdb.db.engine.heavyhitter.QueryHeavyHitters;
+import org.apache.iotdb.db.exception.metadata.MetadataException;
+import org.apache.iotdb.db.metadata.PartialPath;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class HashMapHitter implements QueryHeavyHitters {
+
+ private static final Logger logger =
LoggerFactory.getLogger(HashMapHitter.class);
+ private Map<PartialPath, Integer> counter = new HashMap<>();
+
+ public HashMapHitter(int maxHitterNum) {
+
+ }
+
+ @Override
+ public void acceptQuerySeries(PartialPath queryPath) {
+ counter.put(queryPath, counter.getOrDefault(queryPath, 0) + 1);
+ }
+
+ @Override
+ public List<PartialPath> getTopCompactionSeries(PartialPath sgName) throws
MetadataException {
+ return null;
+ }
+
+ /**
+ * used to persist query frequency
+ *
+ * @param outputPath dump file name
+ */
+ public void printMapToFile(File outputPath) {
+ try (BufferedWriter csvWriter = new BufferedWriter(
+ new OutputStreamWriter(new FileOutputStream(outputPath),
StandardCharsets.UTF_8), 1024)) {
+ File parent = outputPath.getParentFile();
+ if (parent != null && !parent.exists()) {
+ parent.mkdirs();
+ }
+ outputPath.createNewFile();
+ // 写入文件内容
+ for (Map.Entry<PartialPath, Integer> entry : counter.entrySet()) {
+ String line = entry.getKey().getFullPath() + "," + entry.getValue();
+ csvWriter.write(line);
+ csvWriter.newLine();
+ }
+ csvWriter.flush();
+ } catch (Exception e) {
+
+ }
+ }
+}
diff --git
a/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java
b/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java
index d7863c8..f6f7157 100644
---
a/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java
+++
b/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java
@@ -27,6 +27,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.iotdb.db.auth.AuthException;
+import org.apache.iotdb.db.engine.heavyhitter.QueryHitterManager;
import org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.db.exception.query.PathNumOverLimitException;
import org.apache.iotdb.db.exception.query.LogicalOperatorException;
@@ -679,6 +680,11 @@ public class PhysicalGenerator {
List<TSDataType> dataTypes = getSeriesTypes(paths);
queryPlan.setDataTypes(dataTypes);
+ // add query to hitter
+ for (PartialPath path: paths) {
+ QueryHitterManager.getQueryHitter().acceptQuerySeries(path);
+ }
+
// deduplicate from here
if (queryPlan instanceof AlignByDevicePlan) {
return;
