[
https://issues.apache.org/jira/browse/CARBONDATA-204?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15460922#comment-15460922
]
ASF GitHub Bot commented on CARBONDATA-204:
-------------------------------------------
Github user Vimal-Das commented on a diff in the pull request:
https://github.com/apache/incubator-carbondata/pull/123#discussion_r77435578
--- Diff:
core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/DriverQueryStatisticsRecorder.java
---
@@ -78,106 +82,142 @@ public synchronized void
recordStatisticsForDriver(QueryStatistic statistic, Str
*/
public void logStatisticsAsTableDriver() {
synchronized (lock) {
- String tableInfo = collectDriverStatistics();
- if (null != tableInfo) {
- LOGGER.statistic(tableInfo);
+ for (String key: queryStatisticsMap.keySet()) {
+ // print
sql_parse_t,load_meta_t,block_allocation_t,block_identification_t
+ // or just print block_allocation_t,block_identification_t
+ if (queryStatisticsMap.get(key).size() >= 2) {
+ String tableInfo = collectDriverStatistics(key);
+ if (null != tableInfo) {
+ LOGGER.statistic(tableInfo);
+ }
+ }
+ // clear timeout query statistics
+ if(StringUtils.isEmpty(key)) {
+ queryStatisticsMap.remove(key);
+ } else {
+ long interval = System.nanoTime() - Long.parseLong(key);
+ if (interval >
QueryStatisticsConstants.CLEAR_STATISTICS_TIMEOUT) {
+ queryStatisticsMap.remove(key);
+ }
+ }
}
}
}
/**
* Below method will parse queryStatisticsMap and put time into table
*/
- public String collectDriverStatistics() {
- for (String key: queryStatisticsMap.keySet()) {
- try {
- // TODO: get the finished query, and print Statistics
- if (queryStatisticsMap.get(key).size() > 3) {
- String sql_parse_time = "";
- String load_meta_time = "";
- String block_allocation_time = "";
- String block_identification_time = "";
- Double driver_part_time_tmp = 0.0;
- String splitChar = " ";
- // get statistic time from the QueryStatistic
- for (QueryStatistic statistic : queryStatisticsMap.get(key)) {
- switch (statistic.getMessage()) {
- case QueryStatisticsConstants.SQL_PARSE:
- sql_parse_time += statistic.getTimeTaken() + splitChar;
- driver_part_time_tmp += statistic.getTimeTaken();
- break;
- case QueryStatisticsConstants.LOAD_META:
- load_meta_time += statistic.getTimeTaken() + splitChar;
- driver_part_time_tmp += statistic.getTimeTaken();
- break;
- case QueryStatisticsConstants.BLOCK_ALLOCATION:
- block_allocation_time += statistic.getTimeTaken() +
splitChar;
- driver_part_time_tmp += statistic.getTimeTaken();
- break;
- case QueryStatisticsConstants.BLOCK_IDENTIFICATION:
- block_identification_time += statistic.getTimeTaken() +
splitChar;
- driver_part_time_tmp += statistic.getTimeTaken();
- break;
- default:
- break;
- }
- }
- String driver_part_time = driver_part_time_tmp + splitChar;
- // structure the query statistics info table
- StringBuilder tableInfo = new StringBuilder();
- int len1 = 8;
- int len2 = 20;
- int len3 = 21;
- int len4 = 22;
- String line = "+" + printLine("-", len1) + "+" + printLine("-",
len2) + "+" +
- printLine("-", len3) + "+" + printLine("-", len4) + "+";
- String line2 = "|" + printLine(" ", len1) + "+" + printLine("-",
len2) + "+" +
- printLine(" ", len3) + "+" + printLine("-", len4) + "+";
- // table header
- tableInfo.append(line).append("\n");
- tableInfo.append("|" + printLine(" ", (len1 -
"Module".length())) + "Module" + "|" +
- printLine(" ", (len2 - "Operation Step".length())) +
"Operation Step" + "|" +
- printLine(" ", (len3 + len4 + 1 - "Query Cost".length())) +
- "Query Cost" + "|" + "\n");
- // driver part
- tableInfo.append(line).append("\n");
- tableInfo.append("|" + printLine(" ", len1) + "|" +
- printLine(" ", (len2 - "SQL parse".length())) + "SQL parse"
+ "|" +
- printLine(" ", len3) + "|" +
- printLine(" ", (len4 - sql_parse_time.length())) +
sql_parse_time + "|" + "\n");
- tableInfo.append(line2).append("\n");
- tableInfo.append("|" +printLine(" ", (len1 - "Driver".length()))
+ "Driver" + "|" +
- printLine(" ", (len2 - "Load meta data".length())) + "Load
meta data" + "|" +
- printLine(" ", (len3 - driver_part_time.length())) +
driver_part_time + "|" +
- printLine(" ", (len4 - load_meta_time.length())) +
- load_meta_time + "|" + "\n");
- tableInfo.append(line2).append("\n");
- tableInfo.append("|" +
- printLine(" ", (len1 - "Part".length())) + "Part" + "|" +
- printLine(" ", (len2 - "Block allocation".length())) +
- "Block allocation" + "|" +
- printLine(" ", len3) + "|" +
- printLine(" ", (len4 - block_allocation_time.length())) +
- block_allocation_time + "|" + "\n");
- tableInfo.append(line2).append("\n");
- tableInfo.append("|" +
- printLine(" ", len1) + "|" +
- printLine(" ", (len2 - "Block identification".length())) +
- "Block identification" + "|" +
- printLine(" ", len3) + "|" +
- printLine(" ", (len4 - block_identification_time.length())) +
- block_identification_time + "|" + "\n");
- tableInfo.append(line).append("\n");
-
- // once the statistics be printed, remove it from the map
- queryStatisticsMap.remove(key);
- // show query statistic as "query id" + "table"
- return "Print query statistic for query id: " + key + "\n" +
tableInfo.toString();
+ public String collectDriverStatistics(String key) {
+ String sql_parse_time = "";
+ String load_meta_time = "";
+ String block_allocation_time = "";
+ String block_identification_time = "";
+ Double driver_part_time_tmp = 0.0;
+ Double driver_part_time_tmp2 = 0.0;
+ String splitChar = " ";
+ try {
+ // get statistic time from the QueryStatistic
+ for (QueryStatistic statistic : queryStatisticsMap.get(key)) {
--- End diff --
null possibility here also, pass entry.getValue() instead, once you iterate
over entrySet() in the calling method
> Query statistics issue
> -----------------------
>
> Key: CARBONDATA-204
> URL: https://issues.apache.org/jira/browse/CARBONDATA-204
> Project: CarbonData
> Issue Type: Bug
> Reporter: Gin-zhj
> Assignee: Gin-zhj
>
> Query statistics issue:
> 1. some query statistics that never be printed will be keeped into
> querystatisticsMap, so it will cause "out of memory" for long time running
> 2. in some sceniaro, the driver can't record "sql_parse_time" , the driver
> statistics logs will not be output
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
