Github user Vimal-Das commented on a diff in the pull request:
https://github.com/apache/incubator-carbondata/pull/123#discussion_r77435578
--- Diff:
core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/DriverQueryStatisticsRecorder.java
---
@@ -78,106 +82,142 @@ public synchronized void
recordStatisticsForDriver(QueryStatistic statistic, Str
*/
public void logStatisticsAsTableDriver() {
synchronized (lock) {
- String tableInfo = collectDriverStatistics();
- if (null != tableInfo) {
- LOGGER.statistic(tableInfo);
+ for (String key: queryStatisticsMap.keySet()) {
+ // print
sql_parse_t,load_meta_t,block_allocation_t,block_identification_t
+ // or just print block_allocation_t,block_identification_t
+ if (queryStatisticsMap.get(key).size() >= 2) {
+ String tableInfo = collectDriverStatistics(key);
+ if (null != tableInfo) {
+ LOGGER.statistic(tableInfo);
+ }
+ }
+ // clear timeout query statistics
+ if(StringUtils.isEmpty(key)) {
+ queryStatisticsMap.remove(key);
+ } else {
+ long interval = System.nanoTime() - Long.parseLong(key);
+ if (interval >
QueryStatisticsConstants.CLEAR_STATISTICS_TIMEOUT) {
+ queryStatisticsMap.remove(key);
+ }
+ }
}
}
}
/**
* Below method will parse queryStatisticsMap and put time into table
*/
- public String collectDriverStatistics() {
- for (String key: queryStatisticsMap.keySet()) {
- try {
- // TODO: get the finished query, and print Statistics
- if (queryStatisticsMap.get(key).size() > 3) {
- String sql_parse_time = "";
- String load_meta_time = "";
- String block_allocation_time = "";
- String block_identification_time = "";
- Double driver_part_time_tmp = 0.0;
- String splitChar = " ";
- // get statistic time from the QueryStatistic
- for (QueryStatistic statistic : queryStatisticsMap.get(key)) {
- switch (statistic.getMessage()) {
- case QueryStatisticsConstants.SQL_PARSE:
- sql_parse_time += statistic.getTimeTaken() + splitChar;
- driver_part_time_tmp += statistic.getTimeTaken();
- break;
- case QueryStatisticsConstants.LOAD_META:
- load_meta_time += statistic.getTimeTaken() + splitChar;
- driver_part_time_tmp += statistic.getTimeTaken();
- break;
- case QueryStatisticsConstants.BLOCK_ALLOCATION:
- block_allocation_time += statistic.getTimeTaken() +
splitChar;
- driver_part_time_tmp += statistic.getTimeTaken();
- break;
- case QueryStatisticsConstants.BLOCK_IDENTIFICATION:
- block_identification_time += statistic.getTimeTaken() +
splitChar;
- driver_part_time_tmp += statistic.getTimeTaken();
- break;
- default:
- break;
- }
- }
- String driver_part_time = driver_part_time_tmp + splitChar;
- // structure the query statistics info table
- StringBuilder tableInfo = new StringBuilder();
- int len1 = 8;
- int len2 = 20;
- int len3 = 21;
- int len4 = 22;
- String line = "+" + printLine("-", len1) + "+" + printLine("-",
len2) + "+" +
- printLine("-", len3) + "+" + printLine("-", len4) + "+";
- String line2 = "|" + printLine(" ", len1) + "+" + printLine("-",
len2) + "+" +
- printLine(" ", len3) + "+" + printLine("-", len4) + "+";
- // table header
- tableInfo.append(line).append("\n");
- tableInfo.append("|" + printLine(" ", (len1 -
"Module".length())) + "Module" + "|" +
- printLine(" ", (len2 - "Operation Step".length())) +
"Operation Step" + "|" +
- printLine(" ", (len3 + len4 + 1 - "Query Cost".length())) +
- "Query Cost" + "|" + "\n");
- // driver part
- tableInfo.append(line).append("\n");
- tableInfo.append("|" + printLine(" ", len1) + "|" +
- printLine(" ", (len2 - "SQL parse".length())) + "SQL parse"
+ "|" +
- printLine(" ", len3) + "|" +
- printLine(" ", (len4 - sql_parse_time.length())) +
sql_parse_time + "|" + "\n");
- tableInfo.append(line2).append("\n");
- tableInfo.append("|" +printLine(" ", (len1 - "Driver".length()))
+ "Driver" + "|" +
- printLine(" ", (len2 - "Load meta data".length())) + "Load
meta data" + "|" +
- printLine(" ", (len3 - driver_part_time.length())) +
driver_part_time + "|" +
- printLine(" ", (len4 - load_meta_time.length())) +
- load_meta_time + "|" + "\n");
- tableInfo.append(line2).append("\n");
- tableInfo.append("|" +
- printLine(" ", (len1 - "Part".length())) + "Part" + "|" +
- printLine(" ", (len2 - "Block allocation".length())) +
- "Block allocation" + "|" +
- printLine(" ", len3) + "|" +
- printLine(" ", (len4 - block_allocation_time.length())) +
- block_allocation_time + "|" + "\n");
- tableInfo.append(line2).append("\n");
- tableInfo.append("|" +
- printLine(" ", len1) + "|" +
- printLine(" ", (len2 - "Block identification".length())) +
- "Block identification" + "|" +
- printLine(" ", len3) + "|" +
- printLine(" ", (len4 - block_identification_time.length())) +
- block_identification_time + "|" + "\n");
- tableInfo.append(line).append("\n");
-
- // once the statistics be printed, remove it from the map
- queryStatisticsMap.remove(key);
- // show query statistic as "query id" + "table"
- return "Print query statistic for query id: " + key + "\n" +
tableInfo.toString();
+ public String collectDriverStatistics(String key) {
+ String sql_parse_time = "";
+ String load_meta_time = "";
+ String block_allocation_time = "";
+ String block_identification_time = "";
+ Double driver_part_time_tmp = 0.0;
+ Double driver_part_time_tmp2 = 0.0;
+ String splitChar = " ";
+ try {
+ // get statistic time from the QueryStatistic
+ for (QueryStatistic statistic : queryStatisticsMap.get(key)) {
--- End diff --
null possibility here also, pass entry.getValue() instead, once you iterate
over entrySet() in the calling method
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
