This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new 93285f1fd [KYUUBI #6574] Skip eagerly execute command in PlanOnly mode
of Spark Engine
93285f1fd is described below
commit 93285f1fdbb334db5c0b50c1e1d426e64c016b8b
Author: wforget <[email protected]>
AuthorDate: Mon Aug 5 16:04:42 2024 +0800
[KYUUBI #6574] Skip eagerly execute command in PlanOnly mode of Spark Engine
# :mag: Description
## Issue References ๐
This pull request fixes #6574
## Describe Your Solution ๐ง
Skip eagerly execute command in physical and execution plan only mode
## Types of changes :bookmark:
- [ ] Bugfix (non-breaking change which fixes an issue)
- [X] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to change)
## Test Plan ๐งช
#### Related Unit Tests
added unit test
---
# Checklist ๐
- [X] This patch was not authored or co-authored using [Generative
Tooling](https://www.apache.org/legal/generative-tooling.html)
**Be nice. Be informative.**
Closes #6575 from wForget/KYUUBI-6574.
Closes #6574
6f79228c6 [wforget] fix
9aff4a803 [wforget] fix
839ea4a4f [wforget] fix
8a08c9fa7 [wforget] [KYUUBI #6574] Skip eagerly execute command in PlanOnly
mode of Spark Engine
Authored-by: wforget <[email protected]>
Signed-off-by: Cheng Pan <[email protected]>
---
.../engine/spark/operation/PlanOnlyStatement.scala | 9 +++----
.../kyuubi/operation/PlanOnlyOperationSuite.scala | 28 +++++++++++++++++++++-
2 files changed, 32 insertions(+), 5 deletions(-)
diff --git
a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/PlanOnlyStatement.scala
b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/PlanOnlyStatement.scala
index f2a670471..42837120e 100644
---
a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/PlanOnlyStatement.scala
+++
b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/PlanOnlyStatement.scala
@@ -22,6 +22,7 @@ import com.fasterxml.jackson.module.scala.DefaultScalaModule
import org.apache.spark.kyuubi.SparkUtilsHelper
import org.apache.spark.sql.{Row, SparkSession}
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.execution.CommandExecutionMode
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types.StructType
@@ -115,10 +116,10 @@ class PlanOnlyStatement(
SQLConf.get.maxToStringFields,
printOperatorId = false))))
case PhysicalMode =>
- val physical = spark.sql(statement).queryExecution.sparkPlan
+ val physical = spark.sessionState.executePlan(plan,
CommandExecutionMode.SKIP).sparkPlan
iter = new IterableFetchIterator(Seq(Row(physical.toString())))
case ExecutionMode =>
- val executed = spark.sql(statement).queryExecution.executedPlan
+ val executed = spark.sessionState.executePlan(plan,
CommandExecutionMode.SKIP).executedPlan
iter = new IterableFetchIterator(Seq(Row(executed.toString())))
case LineageMode =>
val result = parseLineage(spark, plan)
@@ -142,10 +143,10 @@ class PlanOnlyStatement(
val optimized = spark.sessionState.optimizer.execute(analyzed)
iter = new IterableFetchIterator(Seq(Row(optimized.toJSON)))
case PhysicalMode =>
- val physical = spark.sql(statement).queryExecution.sparkPlan
+ val physical = spark.sessionState.executePlan(plan,
CommandExecutionMode.SKIP).sparkPlan
iter = new IterableFetchIterator(Seq(Row(physical.toJSON)))
case ExecutionMode =>
- val executed = spark.sql(statement).queryExecution.executedPlan
+ val executed = spark.sessionState.executePlan(plan,
CommandExecutionMode.SKIP).executedPlan
iter = new IterableFetchIterator(Seq(Row(executed.toJSON)))
case LineageMode =>
val result = parseLineage(spark, plan)
diff --git
a/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/PlanOnlyOperationSuite.scala
b/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/PlanOnlyOperationSuite.scala
index 2c27b51db..98cbdf41e 100644
---
a/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/PlanOnlyOperationSuite.scala
+++
b/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/PlanOnlyOperationSuite.scala
@@ -233,8 +233,34 @@ class PlanOnlyOperationSuite extends WithKyuubiServer with
HiveJDBCTestHelper {
}
}
+ test("KYUUBI #6574: Skip eagerly execute command in physical/execution plan
only mode") {
+ withJdbcStatement() { statement =>
+ val table = "test_plan_only"
+ val createTableCommand = s"create table $table(i int) using parquet"
+
+ statement.execute(s"SET
${KyuubiConf.OPERATION_PLAN_ONLY_MODE.key}=${PhysicalMode.name}")
+ val physicalPlan = getOperationPlanWithStatement(statement,
createTableCommand)
+ assert(physicalPlan.startsWith("Execute CreateDataSourceTableCommand")
+ && physicalPlan.contains(table))
+
+ statement.execute(s"SET
${KyuubiConf.OPERATION_PLAN_ONLY_MODE.key}=${ExecutionMode.name}")
+ val executionPlan = getOperationPlanWithStatement(statement,
createTableCommand)
+ assert(executionPlan.startsWith("Execute CreateDataSourceTableCommand")
+ && physicalPlan.contains(table))
+
+ statement.execute(s"SET
${KyuubiConf.OPERATION_PLAN_ONLY_MODE.key}=${NoneMode.name}")
+ val e = intercept[KyuubiSQLException](statement.executeQuery(s"select *
from $table"))
+ assert(e.getMessage.contains("TABLE_OR_VIEW_NOT_FOUND")
+ || e.getMessage.contains("Table or view not found"))
+ }
+ }
+
private def getOperationPlanWithStatement(statement: Statement): String = {
- val resultSet = statement.executeQuery("select 1 where true")
+ getOperationPlanWithStatement(statement, "select 1 where true")
+ }
+
+ private def getOperationPlanWithStatement(statement: Statement, sql:
String): String = {
+ val resultSet = statement.executeQuery(sql)
assert(resultSet.next())
resultSet.getString(1)
}
