clear query statistics map
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/60fde412 Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/60fde412 Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/60fde412 Branch: refs/heads/branch-0.1 Commit: 60fde4123514c46883f8d1f131d9c0e518b1e981 Parents: e78106a Author: foryou2030 <foryou2...@126.com> Authored: Fri Sep 2 18:22:03 2016 +0800 Committer: ravipesala <ravi.pes...@gmail.com> Committed: Thu Sep 22 09:57:11 2016 +0530 ---------------------------------------------------------------------- .../DriverQueryStatisticsRecorder.java | 234 +++++++++++-------- .../QueryStatisticsConstants.java | 3 + .../apache/carbondata/core/util/CarbonUtil.java | 14 ++ 3 files changed, 158 insertions(+), 93 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/60fde412/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/DriverQueryStatisticsRecorder.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/DriverQueryStatisticsRecorder.java b/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/DriverQueryStatisticsRecorder.java index 9e83c22..534852f 100644 --- a/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/DriverQueryStatisticsRecorder.java +++ b/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/DriverQueryStatisticsRecorder.java @@ -19,15 +19,19 @@ package org.apache.carbondata.core.carbon.querystatistics; import java.util.ArrayList; -import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; +import static org.apache.carbondata.core.util.CarbonUtil.add; import static org.apache.carbondata.core.util.CarbonUtil.printLine; +import org.apache.commons.lang3.StringUtils; + /** * Class will be used to record and log the query statistics */ @@ -47,7 +51,8 @@ public class DriverQueryStatisticsRecorder { private static final Object lock = new Object(); private DriverQueryStatisticsRecorder() { - queryStatisticsMap = new HashMap<String, List<QueryStatistic>>(); + // use ConcurrentHashMap, it is thread-safe + queryStatisticsMap = new ConcurrentHashMap<String, List<QueryStatistic>>(); } private static DriverQueryStatisticsRecorder carbonLoadStatisticsImplInstance = @@ -78,9 +83,32 @@ public class DriverQueryStatisticsRecorder { */ public void logStatisticsAsTableDriver() { synchronized (lock) { - String tableInfo = collectDriverStatistics(); - if (null != tableInfo) { - LOGGER.statistic(tableInfo); + Iterator<Map.Entry<String, List<QueryStatistic>>> entries = + queryStatisticsMap.entrySet().iterator(); + while (entries.hasNext()) { + Map.Entry<String, List<QueryStatistic>> entry = entries.next(); + String queryId = entry.getKey(); + // clear the unknown query statistics + if(StringUtils.isEmpty(queryId)) { + entries.remove(); + } else { + // clear the timeout query statistics + long interval = System.nanoTime() - Long.parseLong(queryId); + if (interval > QueryStatisticsConstants.CLEAR_STATISTICS_TIMEOUT) { + entries.remove(); + } else { + // print sql_parse_t,load_meta_t,block_allocation_t,block_identification_t + // or just print block_allocation_t,block_identification_t + if (entry.getValue().size() >= 2) { + String tableInfo = collectDriverStatistics(entry.getValue(), queryId); + if (null != tableInfo) { + LOGGER.statistic(tableInfo); + // clear the statistics that has been printed + entries.remove(); + } + } + } + } } } } @@ -88,96 +116,116 @@ public class DriverQueryStatisticsRecorder { /** * Below method will parse queryStatisticsMap and put time into table */ - public String collectDriverStatistics() { - for (String key: queryStatisticsMap.keySet()) { - try { - // TODO: get the finished query, and print Statistics - if (queryStatisticsMap.get(key).size() > 3) { - String sql_parse_time = ""; - String load_meta_time = ""; - String block_allocation_time = ""; - String block_identification_time = ""; - Double driver_part_time_tmp = 0.0; - String splitChar = " "; - // get statistic time from the QueryStatistic - for (QueryStatistic statistic : queryStatisticsMap.get(key)) { - switch (statistic.getMessage()) { - case QueryStatisticsConstants.SQL_PARSE: - sql_parse_time += statistic.getTimeTaken() + splitChar; - driver_part_time_tmp += statistic.getTimeTaken(); - break; - case QueryStatisticsConstants.LOAD_META: - load_meta_time += statistic.getTimeTaken() + splitChar; - driver_part_time_tmp += statistic.getTimeTaken(); - break; - case QueryStatisticsConstants.BLOCK_ALLOCATION: - block_allocation_time += statistic.getTimeTaken() + splitChar; - driver_part_time_tmp += statistic.getTimeTaken(); - break; - case QueryStatisticsConstants.BLOCK_IDENTIFICATION: - block_identification_time += statistic.getTimeTaken() + splitChar; - driver_part_time_tmp += statistic.getTimeTaken(); - break; - default: - break; - } - } - String driver_part_time = driver_part_time_tmp + splitChar; - // structure the query statistics info table - StringBuilder tableInfo = new StringBuilder(); - int len1 = 8; - int len2 = 20; - int len3 = 21; - int len4 = 22; - String line = "+" + printLine("-", len1) + "+" + printLine("-", len2) + "+" + - printLine("-", len3) + "+" + printLine("-", len4) + "+"; - String line2 = "|" + printLine(" ", len1) + "+" + printLine("-", len2) + "+" + - printLine(" ", len3) + "+" + printLine("-", len4) + "+"; - // table header - tableInfo.append(line).append("\n"); - tableInfo.append("|" + printLine(" ", (len1 - "Module".length())) + "Module" + "|" + - printLine(" ", (len2 - "Operation Step".length())) + "Operation Step" + "|" + - printLine(" ", (len3 + len4 + 1 - "Query Cost".length())) + - "Query Cost" + "|" + "\n"); - // driver part - tableInfo.append(line).append("\n"); - tableInfo.append("|" + printLine(" ", len1) + "|" + - printLine(" ", (len2 - "SQL parse".length())) + "SQL parse" + "|" + - printLine(" ", len3) + "|" + - printLine(" ", (len4 - sql_parse_time.length())) + sql_parse_time + "|" + "\n"); - tableInfo.append(line2).append("\n"); - tableInfo.append("|" +printLine(" ", (len1 - "Driver".length())) + "Driver" + "|" + - printLine(" ", (len2 - "Load meta data".length())) + "Load meta data" + "|" + - printLine(" ", (len3 - driver_part_time.length())) + driver_part_time + "|" + - printLine(" ", (len4 - load_meta_time.length())) + - load_meta_time + "|" + "\n"); - tableInfo.append(line2).append("\n"); - tableInfo.append("|" + - printLine(" ", (len1 - "Part".length())) + "Part" + "|" + - printLine(" ", (len2 - "Block allocation".length())) + - "Block allocation" + "|" + - printLine(" ", len3) + "|" + - printLine(" ", (len4 - block_allocation_time.length())) + - block_allocation_time + "|" + "\n"); - tableInfo.append(line2).append("\n"); - tableInfo.append("|" + - printLine(" ", len1) + "|" + - printLine(" ", (len2 - "Block identification".length())) + - "Block identification" + "|" + - printLine(" ", len3) + "|" + - printLine(" ", (len4 - block_identification_time.length())) + - block_identification_time + "|" + "\n"); - tableInfo.append(line).append("\n"); - - // once the statistics be printed, remove it from the map - queryStatisticsMap.remove(key); - // show query statistic as "query id" + "table" - return "Print query statistic for query id: " + key + "\n" + tableInfo.toString(); + public String collectDriverStatistics(List<QueryStatistic> statisticsList, String queryId) { + String sql_parse_time = ""; + String load_meta_time = ""; + String block_allocation_time = ""; + String block_identification_time = ""; + Double driver_part_time_tmp = 0.0; + Double driver_part_time_tmp2 = 0.0; + String splitChar = " "; + try { + // get statistic time from the QueryStatistic + for (QueryStatistic statistic : statisticsList) { + switch (statistic.getMessage()) { + case QueryStatisticsConstants.SQL_PARSE: + sql_parse_time += statistic.getTimeTaken() + splitChar; + driver_part_time_tmp = add(driver_part_time_tmp, statistic.getTimeTaken()); + break; + case QueryStatisticsConstants.LOAD_META: + load_meta_time += statistic.getTimeTaken() + splitChar; + driver_part_time_tmp = add(driver_part_time_tmp, statistic.getTimeTaken()); + break; + case QueryStatisticsConstants.BLOCK_ALLOCATION: + block_allocation_time += statistic.getTimeTaken() + splitChar; + driver_part_time_tmp = add(driver_part_time_tmp, statistic.getTimeTaken()); + driver_part_time_tmp2 = add(driver_part_time_tmp2, statistic.getTimeTaken()); + break; + case QueryStatisticsConstants.BLOCK_IDENTIFICATION: + block_identification_time += statistic.getTimeTaken() + splitChar; + driver_part_time_tmp = add(driver_part_time_tmp, statistic.getTimeTaken()); + driver_part_time_tmp2 = add(driver_part_time_tmp2, statistic.getTimeTaken()); + break; + default: + break; } - } catch (Exception ex) { - return "Put statistics into table failed, catch exception: " + ex.getMessage(); } + String driver_part_time = driver_part_time_tmp + splitChar; + // structure the query statistics info table + StringBuilder tableInfo = new StringBuilder(); + int len1 = 8; + int len2 = 20; + int len3 = 21; + int len4 = 22; + String line = "+" + printLine("-", len1) + "+" + printLine("-", len2) + "+" + + printLine("-", len3) + "+" + printLine("-", len4) + "+"; + String line2 = "|" + printLine(" ", len1) + "+" + printLine("-", len2) + "+" + + printLine(" ", len3) + "+" + printLine("-", len4) + "+"; + // table header + tableInfo.append(line).append("\n"); + tableInfo.append("|" + printLine(" ", (len1 - "Module".length())) + "Module" + "|" + + printLine(" ", (len2 - "Operation Step".length())) + "Operation Step" + "|" + + printLine(" ", (len3 + len4 + 1 - "Query Cost".length())) + "Query Cost" + "|" + "\n"); + tableInfo.append(line).append("\n"); + // print sql_parse_t,load_meta_t,block_allocation_t,block_identification_t + if (!StringUtils.isEmpty(sql_parse_time) && + !StringUtils.isEmpty(load_meta_time) && + !StringUtils.isEmpty(block_allocation_time) && + !StringUtils.isEmpty(block_identification_time)) { + tableInfo.append("|" + printLine(" ", len1) + "|" + + printLine(" ", (len2 - "SQL parse".length())) + "SQL parse" + "|" + + printLine(" ", len3) + "|" + + printLine(" ", (len4 - sql_parse_time.length())) + sql_parse_time + "|" + "\n"); + tableInfo.append(line2).append("\n"); + tableInfo.append("|" + printLine(" ", (len1 - "Driver".length())) + "Driver" + "|" + + printLine(" ", (len2 - "Load meta data".length())) + "Load meta data" + "|" + + printLine(" ", (len3 - driver_part_time.length())) + driver_part_time + "|" + + printLine(" ", (len4 - load_meta_time.length())) + + load_meta_time + "|" + "\n"); + tableInfo.append(line2).append("\n"); + tableInfo.append("|" + printLine(" ", (len1 - "Part".length())) + "Part" + "|" + + printLine(" ", (len2 - "Block allocation".length())) + "Block allocation" + "|" + + printLine(" ", len3) + "|" + + printLine(" ", (len4 - block_allocation_time.length())) + + block_allocation_time + "|" + "\n"); + tableInfo.append(line2).append("\n"); + tableInfo.append("|" + + printLine(" ", len1) + "|" + + printLine(" ", (len2 - "Block identification".length())) + + "Block identification" + "|" + + printLine(" ", len3) + "|" + + printLine(" ", (len4 - block_identification_time.length())) + + block_identification_time + "|" + "\n"); + tableInfo.append(line).append("\n"); + + // show query statistic as "query id" + "table" + return "Print query statistic for query id: " + queryId + "\n" + tableInfo.toString(); + } else if (!StringUtils.isEmpty(block_allocation_time) && + !StringUtils.isEmpty(block_identification_time)) { + // when we can't get sql parse time, we only print the last two + driver_part_time = driver_part_time_tmp2 + splitChar; + tableInfo.append("|" + printLine(" ", (len1 - "Driver".length())) + "Driver" + "|" + + printLine(" ", (len2 - "Block allocation".length())) + "Block allocation" + "|" + + printLine(" ", (len3 - driver_part_time.length())) + driver_part_time + "|" + + printLine(" ", (len4 - block_allocation_time.length())) + + block_allocation_time + "|" + "\n"); + tableInfo.append(line2).append("\n"); + tableInfo.append("|" + + printLine(" ", (len1 - "Part".length())) + "Part" + "|" + + printLine(" ", (len2 - "Block identification".length())) + + "Block identification" + "|" + + printLine(" ", len3) + "|" + + printLine(" ", (len4 - block_identification_time.length())) + + block_identification_time + "|" + "\n"); + tableInfo.append(line).append("\n"); + + // show query statistic as "query id" + "table" + return "Print query statistic for query id: " + queryId + "\n" + tableInfo.toString(); + } + + return null; + } catch (Exception ex) { + return "Put statistics into table failed, catch exception: " + ex.getMessage(); } - return null; } } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/60fde412/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsConstants.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsConstants.java b/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsConstants.java index 540cf1e..118dadb 100644 --- a/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsConstants.java +++ b/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsConstants.java @@ -48,6 +48,9 @@ public interface QueryStatisticsConstants { String RESULT_SIZE = "The size of query result"; + // clear no-use statistics timeout + long CLEAR_STATISTICS_TIMEOUT = 60 * 1000 * 1000000L; + } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/60fde412/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java index 77d041c..c98456e 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java @@ -28,6 +28,7 @@ import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStreamReader; +import java.math.BigDecimal; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.nio.charset.Charset; @@ -1430,5 +1431,18 @@ public final class CarbonUtil { } return builder.toString(); } + + /** + * Below method will for double plus double + * + * @param v1 + * @param v2 + */ + public static double add(double v1, double v2) + { + BigDecimal b1 = new BigDecimal(Double.toString(v1)); + BigDecimal b2 = new BigDecimal(Double.toString(v2)); + return b1.add(b2).doubleValue(); + } }
