[CARBONDATA-2563][CATALYST] Explain query with Order by operator is fired Spark Job which is increase explain query time even though isQueryStatisticsEnabled is false
Even though isQueryStatisticsEnabled is false which means user doesnt wants to see statistics for explain command, still the engine tries to fetch the paritions information which causes a job execution in case of order by query, this is mainly because spark engine does sampling for defifning certain range within paritions for sorting process. As part of solution the explain command process shall fetch the parition info only if isQueryStatisticsEnabled true. This closes #2974 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/21762c54 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/21762c54 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/21762c54 Branch: refs/heads/branch-1.5 Commit: 21762c541df334816c660eb8e4e26aca1df9aa71 Parents: 93268ea Author: s71955 <[email protected]> Authored: Tue Dec 4 20:48:55 2018 +0530 Committer: Raghunandan S <[email protected]> Committed: Mon Dec 17 18:58:33 2018 +0530 ---------------------------------------------------------------------- .../carbondata/core/profiler/ExplainCollector.java | 6 +++++- .../execution/command/table/CarbonExplainCommand.scala | 11 ++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/21762c54/core/src/main/java/org/apache/carbondata/core/profiler/ExplainCollector.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/profiler/ExplainCollector.java b/core/src/main/java/org/apache/carbondata/core/profiler/ExplainCollector.java index 8513dac..75568e4 100644 --- a/core/src/main/java/org/apache/carbondata/core/profiler/ExplainCollector.java +++ b/core/src/main/java/org/apache/carbondata/core/profiler/ExplainCollector.java @@ -171,7 +171,11 @@ public class ExplainCollector { } public static String getFormatedOutput() { - return get().toString(); + if (null != get()) { + return get().toString(); + } else { + return null; + } } @Override http://git-wip-us.apache.org/repos/asf/carbondata/blob/21762c54/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonExplainCommand.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonExplainCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonExplainCommand.scala index 8939c6a..aa7a541 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonExplainCommand.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonExplainCommand.scala @@ -47,12 +47,17 @@ case class CarbonExplainCommand( } private def collectProfiler(sparkSession: SparkSession): Seq[Row] = { - val queryExecution = - sparkSession.sessionState.executePlan(child.asInstanceOf[ExplainCommand].logicalPlan) try { ExplainCollector.setup() - queryExecution.toRdd.partitions if (ExplainCollector.enabled()) { + val queryExecution = + sparkSession.sessionState.executePlan(child.asInstanceOf[ExplainCommand].logicalPlan) + queryExecution.toRdd.partitions + // For count(*) queries the explain collector will be disabled, so profiler + // informations not required in such scenarios. + if (null == ExplainCollector.getFormatedOutput) { + Seq.empty + } Seq(Row("== CarbonData Profiler ==\n" + ExplainCollector.getFormatedOutput)) } else { Seq.empty
