This is an automated email from the ASF dual-hosted git repository.
cloud-fan pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new ba13ea68e51c [SPARK-58738][SQL] Add parse_sql function for SQL
statement JSON analysis
ba13ea68e51c is described below
commit ba13ea68e51cd217afa3bc8d50c9932582232ae5
Author: Serge Rielau <[email protected]>
AuthorDate: Thu Aug 20 11:30:02 2026 +0800
[SPARK-58738][SQL] Add parse_sql function for SQL statement JSON analysis
### What changes were proposed in this pull request?
Add a built-in scalar SQL function `parse_sql(sqlStmt)` that parse-only
analyzes a SQL statement string (via `SparkSqlParser`, matching the session
parser surface) and returns a compact JSON description:
- `parse_success` and, on failure, a nested STANDARD-format error object
(`errorClass`, `messageTemplate`, `sqlState`, `messageParameters`, plus `line`
/ `position` when available). User-facing parse/scripting errors become JSON;
unexpected internal failures propagate.
- Statement classification from ISO/IEC 9075-2:2023 Table 39
(`statement_identifier`, `statement_code`). Spark-only statements use
product-specific identifiers with append-only negative codes. Unknown plans map
to Unrecognized (code `0`); known query shapes are allowlisted as SELECT.
- Multipart `table_references` and `function_references` for lineage (CTE
names and correlation aliases omitted; tables inside CTE bodies included).
- `select_list` entries with multipart `name` only (no expression text).
- `parameter_markers` (`named` and `unnamed_count`).
The expression lives in `sql/core` (needs `SparkSqlParser`) and is
registered when building session state.
### Why are the changes needed?
Dialect-migration and SQL analysis tooling need an efficient, batchable way
to verify that translated SQL parses and to extract statement kind / lineage
references without requiring catalog objects to exist.
### Does this PR introduce _any_ user-facing change?
Yes. New SQL function (`since 5.0.0`, master-only):
```sql
SELECT parse_sql('SELECT a, b FROM t');
--
{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,
-- "table_references":[["t"]],
-- "select_list":[{"name":["a"]},{"name":["b"]}]}
SELECT get_json_object(parse_sql('SELEC'), '$.error.errorClass');
-- PARSE_SYNTAX_ERROR
```
### How was this patch tested?
- `ParseSqlResultSuite` / `ParseSqlSuite` (sql/core)
- Golden file `sql-tests/inputs/parse-sql.sql` (+ analyzer results)
- `ExpressionInfoSuite` / `ExpressionsSchemaSuite`
```
build/sbt 'sql/testOnly *ParseSqlResultSuite *ParseSqlSuite'
SPARK_GENERATE_GOLDEN_FILES=1 build/sbt 'sql/testOnly *SQLQueryTestSuite --
-z parse-sql.sql'
build/sbt 'sql/testOnly *ExpressionsSchemaSuite *ExpressionInfoSuite'
```
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Cursor Grok 4.5
Closes #57962 from srielau/SPARK-58738.
Authored-by: Serge Rielau <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
(cherry picked from commit ebb49417ecdf34ac59c54023309c18af4155bf3c)
Signed-off-by: Wenchen Fan <[email protected]>
---
.../sql/catalyst/analysis/FunctionRegistry.scala | 23 +-
.../org/apache/spark/sql/internal/SQLConf.scala | 12 +
.../spark/sql/catalyst/expressions/ParseSql.scala | 113 ++++
.../spark/sql/catalyst/parser/ParseSqlResult.scala | 426 +++++++++++++++
.../sql/catalyst/parser/SqlStatementCodes.scala | 214 ++++++++
.../sql/internal/BaseSessionStateBuilder.scala | 9 +-
.../sql-functions/sql-expression-schema.md | 1 +
.../analyzer-results/parse-sql-gating.sql.out | 14 +
.../sql-tests/analyzer-results/parse-sql.sql.out | 599 +++++++++++++++++++++
.../sql-tests/inputs/parse-sql-gating.sql | 4 +
.../test/resources/sql-tests/inputs/parse-sql.sql | 249 +++++++++
.../sql-tests/results/parse-sql-gating.sql.out | 16 +
.../resources/sql-tests/results/parse-sql.sql.out | 571 ++++++++++++++++++++
.../apache/spark/sql/ExpressionsSchemaSuite.scala | 16 +-
.../sql/catalyst/expressions/ParseSqlSuite.scala | 90 ++++
.../sql/catalyst/parser/ParseSqlResultSuite.scala | 146 +++++
.../execution/command/ShowFunctionsSuiteBase.scala | 3 +
.../sql/expressions/ExpressionInfoSuite.scala | 2 +
sql/gen-sql-functions-docs.py | 2 +
19 files changed, 2501 insertions(+), 9 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
index 5eef532ab20b..0f091af61671 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
@@ -1062,7 +1062,28 @@ object FunctionRegistry {
fr
}
- val functionSet: Set[FunctionIdentifier] = builtin.listFunction().toSet
+ /**
+ * Builtin function identifiers known to SHOW FUNCTIONS / SessionCatalog.
+ * Starts as the catalyst [[builtin]] set; sql/core-only builtins (e.g.
parse_sql)
+ * are added later via [[registerExtraBuiltin]].
+ */
+ @volatile private var _functionSet: Set[FunctionIdentifier] =
builtin.listFunction().toSet
+
+ def functionSet: Set[FunctionIdentifier] = _functionSet
+
+ /**
+ * Registers a builtin that cannot live in catalyst (e.g. depends on
SparkSqlParser
+ * in sql/core). Updates both [[builtin]] (so session clones / catalog reset
see it)
+ * and [[functionSet]] (so SHOW FUNCTIONS classifies it as SYSTEM, not USER).
+ */
+ private[sql] def registerExtraBuiltin(
+ name: String,
+ info: ExpressionInfo,
+ builder: FunctionBuilder): Unit = synchronized {
+ val id = builtinFunctionIdentifier(name)
+ builtin.registerFunction(id, info, builder)
+ _functionSet = _functionSet + id
+ }
/** Registry for internal functions used by Connect and the Column API. */
private[sql] val internal: SimpleFunctionRegistry =
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 6e960452cd04..64ee81555734 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -5672,6 +5672,16 @@ object SQLConf {
.booleanConf
.createWithDefault(false)
+ val PARSE_SQL_ENABLED =
+ buildConf("spark.sql.function.parseSql.enabled")
+ .doc("When true, enables the parse_sql SQL function. This feature is
under active " +
+ "development; the JSON contract may change across releases while the
flag remains " +
+ "off by default.")
+ .version("4.4.0")
+ .withBindingPolicy(ConfigBindingPolicy.SESSION)
+ .booleanConf
+ .createWithDefault(false)
+
val ELT_OUTPUT_AS_STRING = buildConf("spark.sql.function.eltOutputAsString")
.doc("When this option is set to false and all inputs are binary, `elt`
returns " +
"an output as binary. Otherwise, it returns as a string.")
@@ -9504,6 +9514,8 @@ class SQLConf extends Serializable with Logging with
SqlApiConf {
def concatBinaryAsString: Boolean = getConf(CONCAT_BINARY_AS_STRING)
+ def parseSqlEnabled: Boolean = getConf(PARSE_SQL_ENABLED)
+
def eltOutputAsString: Boolean = getConf(ELT_OUTPUT_AS_STRING)
def validatePartitionColumns: Boolean = getConf(VALIDATE_PARTITION_COLUMNS)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/expressions/ParseSql.scala
b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/expressions/ParseSql.scala
new file mode 100644
index 000000000000..649feeee9adf
--- /dev/null
+++
b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/expressions/ParseSql.scala
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.expressions
+
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry,
FunctionRegistryBase, TypeCheckResult}
+import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
+import org.apache.spark.sql.catalyst.parser.ParseSqlResult
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.types.StringTypeWithCollation
+import org.apache.spark.sql.types.{AbstractDataType, DataType, StringType}
+import org.apache.spark.unsafe.types.UTF8String
+
+/**
+ * Parses a SQL statement string and returns a compact JSON description of the
+ * unresolved statement (identifier/code, lineage references, select-list
names,
+ * parameters), or a STANDARD-format error object when the statement does not
+ * parse.
+ *
+ * Behind [[SQLConf.PARSE_SQL_ENABLED]] while the JSON contract is still
+ * evolving. Designed for batch evaluation over DataFrames of SQL text.
+ * User-facing parse errors become JSON; unexpected internal failures
propagate.
+ */
+// scalastyle:off line.size.limit
+@ExpressionDescription(
+ usage = """_FUNC_(sqlStmt) - Parses `sqlStmt` with the stock Spark SQL
parser and
+ returns a JSON string describing the statement (parse success, Table 39
statement
+ identifier/code, target and source table references for lineage,
select-list column
+ names, and parameter markers). Session parser extensions are not applied.
+ Requires spark.sql.function.parseSql.enabled=true. On syntax / parse error
returns JSON
+ with `parse_success` false, source location, and a nested STANDARD error
object
+ instead of throwing.""",
+ arguments = """
+ Arguments:
+ * sqlStmt - A SQL statement string to parse.
+ An expression that evaluates to a string.
+ """,
+ examples = """
+ Examples:
+ > SELECT _FUNC_('SELECT a, b FROM t');
+
{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"source_table_references":[["t"]],"select_list":[{"name":["a"]},{"name":["b"]}]}
+ > SELECT get_json_object(_FUNC_('SELEC'), '$.error.errorClass');
+ PARSE_SYNTAX_ERROR
+ """,
+ group = "misc_funcs",
+ since = "4.4.0")
+// scalastyle:on line.size.limit
+case class ParseSql(child: Expression)
+ extends UnaryExpression
+ with ImplicitCastInputTypes
+ with CodegenFallback {
+
+ override def prettyName: String = "parse_sql"
+
+ override def nullable: Boolean = true
+
+ override def nullIntolerant: Boolean = true
+
+ override def dataType: DataType = StringType
+
+ override def inputTypes: Seq[AbstractDataType] =
+ Seq(StringTypeWithCollation(supportsTrimCollation = true))
+
+ override def checkInputDataTypes(): TypeCheckResult = {
+ if (!SQLConf.get.parseSqlEnabled) {
+ throw new AnalysisException(
+ errorClass = "FEATURE_NOT_ENABLED",
+ messageParameters = Map(
+ "featureName" -> "parse_sql",
+ "configKey" -> SQLConf.PARSE_SQL_ENABLED.key,
+ "configValue" -> "true"))
+ }
+ super.checkInputDataTypes()
+ }
+
+ override def nullSafeEval(input: Any): Any = {
+ val sql = input.asInstanceOf[UTF8String].toString
+ UTF8String.fromString(ParseSqlResult.fromSql(sql))
+ }
+
+ override protected def withNewChildInternal(newChild: Expression): ParseSql =
+ copy(child = newChild)
+}
+
+object ParseSql {
+ /** Register the builtin with a session function registry and the global
builtin set. */
+ def register(registry: FunctionRegistry): Unit = {
+ val (info, builder) = FunctionRegistryBase.build[ParseSql]("parse_sql",
Some("4.4.0"))
+ // Keep the session registry in sync for the first session (cloned before
this runs).
+ registry.registerFunction(
+ FunctionRegistry.builtinFunctionIdentifier("parse_sql"),
+ info,
+ builder)
+ // Also publish into FunctionRegistry.builtin / functionSet so SHOW
USER/SYSTEM
+ // FUNCTIONS classify parse_sql as SYSTEM rather than a user/temp function.
+ FunctionRegistry.registerExtraBuiltin("parse_sql", info, builder)
+ }
+}
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseSqlResult.scala
b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseSqlResult.scala
new file mode 100644
index 000000000000..228da54c3c7d
--- /dev/null
+++
b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseSqlResult.scala
@@ -0,0 +1,426 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.parser
+
+import scala.collection.mutable
+
+import org.json4s._
+import org.json4s.jackson.JsonMethods.{compact, parse => parseJson, render}
+
+import org.apache.spark.{ErrorMessageFormat, SparkThrowable,
SparkThrowableHelper}
+import org.apache.spark.sql.catalyst.analysis._
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.trees.{CurrentOrigin, Origin,
SQLQueryContext}
+import org.apache.spark.sql.exceptions.SqlScriptingException
+import org.apache.spark.sql.execution.SparkSqlParser
+import org.apache.spark.sql.execution.command.{CreateViewCommand,
DescribeQueryCommand, ExplainCommand}
+import org.apache.spark.sql.execution.datasources.CreateTempViewUsing
+
+/**
+ * Parses a SQL statement string and returns a compact JSON description of the
+ * unresolved plan (parse-only; no catalog resolution).
+ *
+ * Uses a stock [[SparkSqlParser]] (ThreadLocal) so statement coverage matches
+ * the default production parser (EXPLAIN / SET / ADD JAR / temp views / etc.).
+ * Session-specific [[org.apache.spark.sql.SparkSessionExtensions]] parser
+ * wrappers are intentionally not applied: `parse_sql` must evaluate on
+ * executors without a session, so only the stock parser is available under
+ * distributed eval.
+ *
+ * On success the JSON always includes `parse_success`, the statement
+ * identifier/code (ISO/IEC 9075-2:2023 Table 39), and omits unused optional
+ * fields (`target_table_references`, `source_table_references`,
+ * `function_references`, `select_list`, `parameter_markers`) when empty. On
parse
+ * failure it returns `parse_success: false` with source location and a nested
+ * STANDARD-format error object, and does not throw. Only [[ParseException]] /
+ * [[SqlScriptingException]] are converted to JSON; unexpected / internal
+ * failures propagate so the function fails.
+ */
+object ParseSqlResult {
+
+ private val parser: ThreadLocal[SparkSqlParser] =
+ ThreadLocal.withInitial(() => new SparkSqlParser())
+
+ /** Parse `sql` and render the JSON result string. */
+ def fromSql(sql: String): String = {
+ try {
+ // Do not inherit the outer query's origin from the parse_sql expression.
+ // Errors and parsed nodes must refer to the SQL string passed to this
function.
+ val origin = if (sql.nonEmpty) {
+ Origin(startIndex = Some(0), stopIndex = Some(sql.length - 1), sqlText
= Some(sql))
+ } else {
+ Origin(sqlText = Some(sql))
+ }
+ CurrentOrigin.withOrigin(origin) {
+ val plan = parser.get().parsePlan(sql)
+ fromPlan(plan)
+ }
+ } catch {
+ // User-facing parse / scripting failures become JSON; everything else
fails.
+ case e: ParseException =>
+ errorJson(e)
+ case e: SqlScriptingException =>
+ errorJson(e)
+ }
+ }
+
+ /** Build success JSON from an already-parsed unresolved plan. */
+ def fromPlan(plan: LogicalPlan): String = {
+ val classification = SqlStatementCodes.classify(plan)
+ val fields = mutable.ListBuffer.empty[JField]
+ fields += "parse_success" -> JBool(true)
+ fields += "statement_identifier" ->
JString(classification.statementIdentifier)
+ fields += "statement_code" -> JInt(classification.statementCode)
+ // Omit unused collections / markers so consumers can treat absence as
empty.
+ val refs = collectPlanReferences(plan)
+ if (refs.targetTables.nonEmpty) {
+ fields += "target_table_references" ->
+ JArray(refs.targetTables.map(partsToJArray).toList)
+ }
+ if (refs.sourceTables.nonEmpty) {
+ fields += "source_table_references" ->
+ JArray(refs.sourceTables.map(partsToJArray).toList)
+ }
+ if (refs.functions.nonEmpty) {
+ fields += "function_references" ->
JArray(refs.functions.map(partsToJArray).toList)
+ }
+ val selectList = collectSelectList(plan)
+ if (selectList.nonEmpty) {
+ fields += "select_list" -> JArray(selectList.toList)
+ }
+ refs.parameterMarkers.foreach(markers => fields += "parameter_markers" ->
markers)
+ compact(render(JObject(fields.toList)))
+ }
+
+ private def errorJson(e: SparkThrowable with Throwable): String = {
+ val errorObj = parseJson(
+ SparkThrowableHelper.getMessage(e,
ErrorMessageFormat.STANDARD)).asInstanceOf[JObject]
+ val origin = e match {
+ case p: ParseException => Some(p.start)
+ case s: SqlScriptingException => Some(s.origin)
+ case _ => None
+ }
+ val locationFields = origin.toSeq.flatMap(originFields)
+ val contextFields = if (errorObj.obj.exists(_._1 == "queryContext")) {
+ Nil
+ } else {
+ origin.toSeq.flatMap(queryContextField)
+ }
+ compact(render(JObject(
+ "parse_success" -> JBool(false),
+ "error" -> JObject(errorObj.obj ++ contextFields ++ locationFields)
+ )))
+ }
+
+ private def queryContextField(origin: Origin): Option[JField] =
origin.context match {
+ case context: SQLQueryContext if context.isValid =>
+ Some("queryContext" -> JArray(List(JObject(
+ "objectType" -> JString(context.objectType),
+ "objectName" -> JString(context.objectName),
+ "startIndex" -> JInt(context.startIndex + 1),
+ "stopIndex" -> JInt(context.stopIndex + 1),
+ "fragment" -> JString(context.fragment)
+ ))))
+ case _ => None
+ }
+
+ private def originFields(origin: Origin): Seq[JField] = Seq(
+ origin.line.map(line => "line" -> JInt(line)),
+ origin.startPosition.map(position => "position" -> JInt(position))).flatten
+
+ private def partsToJArray(parts: Seq[String]): JArray =
+ JArray(parts.map(JString).toList)
+
+ /**
+ * Walk expressions in all product fields, including wrappers such as column
+ * definitions that [[LogicalPlan.expressions]] does not descend into.
+ */
+ private def foreachExpressionDeep(plan: LogicalPlan)(f: Expression => Unit):
Unit = {
+ def visit(value: Any): Unit = value match {
+ case e: Expression => f(e)
+ case _: LogicalPlan =>
+ case values: Iterable[_] => values.foreach(visit)
+ case value: Product => value.productIterator.foreach(visit)
+ case _ =>
+ }
+ plan.productIterator.foreach(visit)
+ }
+
+ /** Whether a command's single `child` names the statement target
table/view. */
+ private def isTargetTableChild(parent: LogicalPlan): Boolean = parent match {
+ case _: DeleteFromTable | _: DeleteFromTableWithFilters | _: UpdateTable |
+ _: CreateTable | _: ReplaceTable | _: DropTable | _: DropView |
+ _: TruncateTable | _: TruncatePartition | _: AlterTableCommand |
+ _: RenameTable | _: SetViewProperties | _: RefreshTable |
+ _: UncacheTable | _: CommentOnTable | _: CreateIndex =>
+ true
+ case _ => false
+ }
+
+ /** CTE aliases visible at a given point of the walk, normalized for lookup.
*/
+ private type CteScope = Set[String]
+
+ /** Whether a table/view reference names the statement target or a read
source. */
+ private sealed trait TableRefRole
+ private object TableRefRole {
+ case object Target extends TableRefRole
+ case object Source extends TableRefRole
+ }
+
+ private def normalizeCteName(name: String): String =
+ name.toLowerCase(java.util.Locale.ROOT)
+
+ private def childScope(parent: LogicalPlan, scope: CteScope): CteScope =
parent match {
+ case w: UnresolvedWith => scope ++ w.cteRelations.map(r =>
normalizeCteName(r._1))
+ case _ => scope
+ }
+
+ /**
+ * Deep plan walk that hands each node the CTE aliases in scope there and the
+ * table-reference role for relation nodes at that position, and covers tree
+ * slots that standard `collect` / `collectWithSubqueries` miss.
+ */
+ private def foreachPlanDeep(
+ plan: LogicalPlan)(f: (LogicalPlan, CteScope, TableRefRole) => Unit):
Unit =
+ visitPlan(plan, Set.empty, TableRefRole.Source)(f)
+
+ private def visitPlan(
+ plan: LogicalPlan,
+ scope: CteScope,
+ role: TableRefRole)(f: (LogicalPlan, CteScope, TableRefRole) => Unit):
Unit = {
+ f(plan, scope, role)
+ plan match {
+ case s: SingleStatement =>
+ visitPlan(s.parsedPlan, scope, role)(f)
+ case _ =>
+ visitNonChildSlots(plan, scope, role)(f)
+ visitChildPlans(plan, scope, role)(f)
+ }
+ }
+
+ private def visitChildPlans(
+ parent: LogicalPlan,
+ scope: CteScope,
+ role: TableRefRole)(f: (LogicalPlan, CteScope, TableRefRole) => Unit):
Unit = {
+ val nextScope = childScope(parent, scope)
+ parent match {
+ case m: MergeIntoTable =>
+ visitPlan(m.targetTable, scope, TableRefRole.Target)(f)
+ visitPlan(m.sourceTable, scope, TableRefRole.Source)(f)
+ case i: InsertIntoStatement =>
+ visitPlan(i.query, nextScope, TableRefRole.Source)(f)
+ case c: CreateTableAsSelect =>
+ visitPlan(c.name, scope, TableRefRole.Target)(f)
+ visitPlan(c.query, nextScope, TableRefRole.Source)(f)
+ case r: ReplaceTableAsSelect =>
+ visitPlan(r.name, scope, TableRefRole.Target)(f)
+ visitPlan(r.query, nextScope, TableRefRole.Source)(f)
+ case cv: CreateView =>
+ visitPlan(cv.child, scope, TableRefRole.Target)(f)
+ visitPlan(cv.query, nextScope, TableRefRole.Source)(f)
+ case cts: CacheTableAsSelect =>
+ visitPlan(cts.plan, nextScope, TableRefRole.Source)(f)
+ case _: CacheTable =>
+ // Name is taken from multipartIdentifier or the non-child `table`
slot.
+ case SubqueryAlias(_, child) =>
+ visitPlan(child, nextScope, role)(f)
+ case _ =>
+ parent.subqueries.foreach(sq => visitPlan(sq, nextScope,
TableRefRole.Source)(f))
+ parent.children.foreach { child =>
+ val childRole =
+ if (isTargetTableChild(parent)) TableRefRole.Target else role
+ visitPlan(child, nextScope, childRole)(f)
+ }
+ }
+ }
+
+ /** Visit plan slots that are not exposed via `children` / subqueries. */
+ private def visitNonChildSlots(
+ plan: LogicalPlan,
+ scope: CteScope,
+ role: TableRefRole)(f: (LogicalPlan, CteScope, TableRefRole) => Unit):
Unit = {
+ plan match {
+ case w: UnresolvedWith =>
+ // A CTE definition sees the aliases defined before it, plus its own
+ // name when the clause is RECURSIVE. Later aliases are not in scope,
+ // so a definition naming one refers to the real table.
+ var definitionScope = scope
+ w.cteRelations.foreach { case (name, ctePlan, _) =>
+ val normalized = normalizeCteName(name)
+ val bodyScope =
+ if (w.allowRecursion) definitionScope + normalized else
definitionScope
+ visitPlan(ctePlan, bodyScope, TableRefRole.Source)(f)
+ definitionScope += normalized
+ }
+ case i: InsertIntoStatement =>
+ visitPlan(i.table, scope, TableRefRole.Target)(f)
+ case c: CacheTable if c.multipartIdentifier.isEmpty =>
+ visitPlan(c.table, scope, TableRefRole.Target)(f)
+ case c: CompoundBody =>
+ c.handlers.foreach(h => visitPlan(h, scope, TableRefRole.Source)(f))
+ case s: SimpleCaseStatement =>
+ s.elseBody.foreach(b => visitPlan(b, scope, TableRefRole.Source)(f))
+ case ExplainCommand(logicalPlan, _) =>
+ visitPlan(logicalPlan, scope, TableRefRole.Source)(f)
+ case DescribeQueryCommand(_, queryPlan) =>
+ visitPlan(queryPlan, scope, TableRefRole.Source)(f)
+ case _ =>
+ }
+ }
+
+ private def tableIdentifierParts(id:
org.apache.spark.sql.catalyst.TableIdentifier): Seq[String] =
+ id.catalog.toSeq ++ id.database.toSeq :+ id.table
+
+ private final case class PlanReferences(
+ targetTables: Seq[Seq[String]],
+ sourceTables: Seq[Seq[String]],
+ functions: Seq[Seq[String]],
+ parameterMarkers: Option[JObject])
+
+ /**
+ * Collect multipart table/view identifiers, function names, and parameter
+ * markers for lineage in a single deep walk. Target references name the
+ * table/view a DML or DDL statement writes to or alters; source references
+ * name tables read in FROM clauses and query bodies. A single-part name is
+ * dropped only when a CTE alias in scope at that node shadows it. Function /
+ * variable identifiers are not collected as tables. Deduplicates while
+ * preserving first-seen order within each category.
+ */
+ private def collectPlanReferences(plan: LogicalPlan): PlanReferences = {
+ val targetTables = mutable.LinkedHashSet.empty[Seq[String]]
+ val sourceTables = mutable.LinkedHashSet.empty[Seq[String]]
+ val functions = mutable.LinkedHashSet.empty[Seq[String]]
+ val namedParams = mutable.LinkedHashSet.empty[String]
+ var unnamedCount = 0
+
+ def isCteName(parts: Seq[String], scope: CteScope): Boolean = parts match {
+ case Seq(name) => scope.contains(normalizeCteName(name))
+ case _ => false
+ }
+
+ def addTable(parts: Seq[String], scope: CteScope, role: TableRefRole):
Unit = {
+ if (parts.nonEmpty && !isCteName(parts, scope)) {
+ role match {
+ case TableRefRole.Target => targetTables += parts
+ case TableRefRole.Source => sourceTables += parts
+ }
+ }
+ }
+
+ def addFunction(parts: Seq[String]): Unit = {
+ if (parts.nonEmpty) functions += parts
+ }
+
+ def visitExpr(e: Expression): Unit = e.foreach {
+ case f: UnresolvedFunction => addFunction(f.nameParts)
+ case n: NamedParameter => namedParams += n.name
+ case _: PosParameter => unnamedCount += 1
+ case _ =>
+ }
+
+ def addTarget(parts: Seq[String]): Unit = {
+ if (parts.nonEmpty) targetTables += parts
+ }
+
+ foreachPlanDeep(plan) { (p, scope, role) =>
+ foreachExpressionDeep(p)(visitExpr)
+ def add(parts: Seq[String]): Unit = addTable(parts, scope, role)
+ p match {
+ case u: UnresolvedRelation => add(u.multipartIdentifier)
+ case u: UnresolvedTable => add(u.multipartIdentifier)
+ case u: UnresolvedView => add(u.multipartIdentifier)
+ case u: UnresolvedTableOrView => add(u.multipartIdentifier)
+ case u: UnresolvedIdentifier if role == TableRefRole.Target =>
add(u.nameParts)
+ case c: CreateViewCommand => addTarget(tableIdentifierParts(c.name))
+ case c: CreateTempViewUsing =>
addTarget(tableIdentifierParts(c.tableIdent))
+ case c: CacheTable if c.multipartIdentifier.nonEmpty =>
+ addTarget(c.multipartIdentifier)
+ case u: UnresolvedTableValuedFunction => addFunction(u.name)
+ case _ =>
+ }
+ }
+
+ val markers =
+ if (namedParams.isEmpty && unnamedCount == 0) {
+ None
+ } else {
+ val markerFields = mutable.ListBuffer.empty[JField]
+ if (namedParams.nonEmpty) {
+ markerFields += "named" -> JArray(namedParams.toList.map(JString))
+ }
+ if (unnamedCount > 0) markerFields += "unnamed_count" ->
JInt(unnamedCount)
+ Some(JObject(markerFields.toList))
+ }
+ PlanReferences(targetTables.toSeq, sourceTables.toSeq, functions.toSeq,
markers)
+ }
+
+ /**
+ * Collect the primary select list as `{name}` objects (multipart name
+ * parts only). Empty for non-query statements without a projected query
body.
+ */
+ private def collectSelectList(plan: LogicalPlan): Seq[JObject] = {
+ val query = primaryQueryPlan(plan)
+ val named: Seq[NamedExpression] = query match {
+ case p: Project => p.projectList
+ case a: Aggregate => a.aggregateExpressions
+ case _ => Nil
+ }
+ named.map(selectListItem)
+ }
+
+ private def primaryQueryPlan(plan: LogicalPlan): LogicalPlan = plan match {
+ case UnresolvedWith(child, _, _) => primaryQueryPlan(child)
+ case InsertIntoStatement(_, _, _, query, _, _, _, _, _) =>
+ primaryQueryPlan(query)
+ case c: CreateTableAsSelect => primaryQueryPlan(c.query)
+ case r: ReplaceTableAsSelect => primaryQueryPlan(r.query)
+ case c: CreateView => primaryQueryPlan(c.query)
+ case c: CreateViewCommand => primaryQueryPlan(c.plan)
+ case c: CacheTableAsSelect => primaryQueryPlan(c.plan)
+ case ExplainCommand(logicalPlan, _) => primaryQueryPlan(logicalPlan)
+ case DescribeQueryCommand(_, queryPlan) => primaryQueryPlan(queryPlan)
+ case SubqueryAlias(_, child) => primaryQueryPlan(child)
+ case Sort(_, _, child, _) => primaryQueryPlan(child)
+ case Filter(_, child) => primaryQueryPlan(child)
+ case UnresolvedHaving(_, child) => primaryQueryPlan(child)
+ case UnresolvedQualify(_, child) => primaryQueryPlan(child)
+ case Distinct(child) => primaryQueryPlan(child)
+ case GlobalLimit(_, child) => primaryQueryPlan(child)
+ case LocalLimit(_, child) => primaryQueryPlan(child)
+ case Offset(_, child) => primaryQueryPlan(child)
+ case Repartition(_, _, child) => primaryQueryPlan(child)
+ case RepartitionByExpression(_, child, _, _) => primaryQueryPlan(child)
+ case Sample(_, _, _, _, child, _) => primaryQueryPlan(child)
+ case other => other
+ }
+
+ private def selectListItem(ne: NamedExpression): JObject = ne match {
+ case Alias(_, name) =>
+ JObject("name" -> partsToJArray(Seq(name)))
+ case _: UnresolvedAlias =>
+ JObject("name" -> partsToJArray(Nil))
+ case s: UnresolvedStar =>
+ val name = s.target.map(_ :+ "*").getOrElse(Seq("*"))
+ JObject("name" -> partsToJArray(name))
+ case a: UnresolvedAttribute =>
+ JObject("name" -> partsToJArray(a.nameParts))
+ case other =>
+ JObject("name" -> partsToJArray(Seq(other.name)))
+ }
+}
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/parser/SqlStatementCodes.scala
b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/parser/SqlStatementCodes.scala
new file mode 100644
index 000000000000..cb2fae1d9af9
--- /dev/null
+++
b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/parser/SqlStatementCodes.scala
@@ -0,0 +1,214 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.parser
+
+import org.apache.spark.sql.catalyst.analysis.{
+ RelationTimeTravel,
+ ResolvedInlineTable,
+ UnresolvedExecuteImmediate,
+ UnresolvedHaving,
+ UnresolvedInlineTable,
+ UnresolvedRelation,
+ UnresolvedTableValuedFunction
+}
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.execution.command._
+import org.apache.spark.sql.execution.datasources.{CreateTempViewUsing,
RefreshResource}
+import org.apache.spark.sql.metricview.logical.CreateMetricView
+
+/**
+ * Classification of a parsed SQL statement using ISO/IEC 9075-2:2023 Table 39,
+ * "SQL-statement codes" (clause 23.1 <get diagnostics statement>).
+ *
+ * @param statementIdentifier Table 39 Identifier column (or Spark product
name)
+ * @param statementCode Table 39 Code column; Spark-only statements use
negative
+ * implementation-defined codes (Table 39 IE005 / IV190)
+ */
+case class SqlStatementClassification(
+ statementIdentifier: String,
+ statementCode: Int)
+
+/**
+ * Maps unresolved [[LogicalPlan]]s to Table 39 statement codes.
+ *
+ * Spark-only statements use the standard's implementation-defined escape
hatch:
+ * a product-specific identifier and a distinct negative code. Codes are
+ * append-only and must never be renumbered.
+ *
+ * Unknown plans map to [[Unrecognized]] (empty identifier, code 0). Query
+ * shapes are allowlisted; unknown non-commands are not assumed to be SELECT.
+ */
+object SqlStatementCodes {
+
+ // Standard Table 39 entries used by Spark SQL (ISO/IEC 9075-2:2023).
+ val Select: SqlStatementClassification =
SqlStatementClassification("SELECT", 21)
+ val Insert: SqlStatementClassification =
SqlStatementClassification("INSERT", 50)
+ val DeleteWhere: SqlStatementClassification =
SqlStatementClassification("DELETE WHERE", 19)
+ val UpdateWhere: SqlStatementClassification =
SqlStatementClassification("UPDATE WHERE", 82)
+ val Merge: SqlStatementClassification = SqlStatementClassification("MERGE",
128)
+ val CreateTable: SqlStatementClassification =
SqlStatementClassification("CREATE TABLE", 77)
+ val CreateView: SqlStatementClassification =
SqlStatementClassification("CREATE VIEW", 84)
+ val DropTable: SqlStatementClassification = SqlStatementClassification("DROP
TABLE", 32)
+ val DropView: SqlStatementClassification = SqlStatementClassification("DROP
VIEW", 36)
+ val AlterTable: SqlStatementClassification =
SqlStatementClassification("ALTER TABLE", 4)
+ val CreateSchema: SqlStatementClassification =
SqlStatementClassification("CREATE SCHEMA", 64)
+ val DropSchema: SqlStatementClassification =
SqlStatementClassification("DROP SCHEMA", 31)
+ val SetSchema: SqlStatementClassification = SqlStatementClassification("SET
SCHEMA", 74)
+ val TruncateTable: SqlStatementClassification =
+ SqlStatementClassification("TRUNCATE TABLE", 139)
+ val CreateRoutine: SqlStatementClassification =
SqlStatementClassification("CREATE ROUTINE", 14)
+ val DropRoutine: SqlStatementClassification =
SqlStatementClassification("DROP ROUTINE", 30)
+ val ExecuteImmediate: SqlStatementClassification =
+ SqlStatementClassification("EXECUTE IMMEDIATE", 43)
+ val Call: SqlStatementClassification = SqlStatementClassification("CALL", 7)
+
+ // Table 39 "Unrecognized statements": empty identifier, code 0.
+ val Unrecognized: SqlStatementClassification =
SqlStatementClassification("", 0)
+
+ // Spark product-specific identifiers with append-only negative codes
+ // (Table 39 implementation-defined / IE005 row: negative Code values).
+ val CacheTable: SqlStatementClassification = spark("CACHE TABLE", -1)
+ val CacheTableAsSelect: SqlStatementClassification = spark("CACHE TABLE AS
SELECT", -2)
+ val UncacheTable: SqlStatementClassification = spark("UNCACHE TABLE", -3)
+ val RefreshTable: SqlStatementClassification = spark("REFRESH TABLE", -4)
+ val ShowTables: SqlStatementClassification = spark("SHOW TABLES", -5)
+ val DescribeTable: SqlStatementClassification = spark("DESCRIBE TABLE", -6)
+ val AnalyzeTable: SqlStatementClassification = spark("ANALYZE TABLE", -7)
+ val DeclareVariable: SqlStatementClassification = spark("DECLARE VARIABLE",
-8)
+ val SetVariable: SqlStatementClassification = spark("SET VARIABLE", -9)
+ val DropVariable: SqlStatementClassification = spark("DROP VARIABLE", -10)
+ val ShowTableProperties: SqlStatementClassification = spark("SHOW
TBLPROPERTIES", -11)
+ val DescribeNamespace: SqlStatementClassification = spark("DESCRIBE
NAMESPACE", -12)
+ val ShowFunctions: SqlStatementClassification = spark("SHOW FUNCTIONS", -13)
+ val DescribeFunction: SqlStatementClassification = spark("DESCRIBE
FUNCTION", -14)
+ val ShowCreateTable: SqlStatementClassification = spark("SHOW CREATE TABLE",
-15)
+ val ShowColumns: SqlStatementClassification = spark("SHOW COLUMNS", -16)
+ val ShowPartitions: SqlStatementClassification = spark("SHOW PARTITIONS",
-17)
+ val ShowViews: SqlStatementClassification = spark("SHOW VIEWS", -18)
+ val RefreshFunction: SqlStatementClassification = spark("REFRESH FUNCTION",
-19)
+ val CommentOnNamespace: SqlStatementClassification = spark("COMMENT ON
NAMESPACE", -20)
+ val CommentOnTable: SqlStatementClassification = spark("COMMENT ON TABLE",
-21)
+ // SQL/PSM-style scripting (9075-4); not in Foundation Table 39.
+ val BeginEnd: SqlStatementClassification = spark("BEGIN END", -22)
+ // SparkSqlParser-only session / resource commands (append-only).
+ val Explain: SqlStatementClassification = spark("EXPLAIN", -23)
+ val Set: SqlStatementClassification = spark("SET", -24)
+ val Reset: SqlStatementClassification = spark("RESET", -25)
+ val AddJar: SqlStatementClassification = spark("ADD JAR", -26)
+ val AddFile: SqlStatementClassification = spark("ADD FILE", -27)
+ val AddArchive: SqlStatementClassification = spark("ADD ARCHIVE", -28)
+ val ListJar: SqlStatementClassification = spark("LIST JAR", -29)
+ val ListFile: SqlStatementClassification = spark("LIST FILE", -30)
+ val ClearCache: SqlStatementClassification = spark("CLEAR CACHE", -31)
+ val RefreshResourceCmd: SqlStatementClassification = spark("REFRESH
RESOURCE", -32)
+ val DescribeQuery: SqlStatementClassification = spark("DESCRIBE QUERY", -33)
+ val ShowCatalogs: SqlStatementClassification = spark("SHOW CATALOGS", -34)
+ val ShowCurrentNamespace: SqlStatementClassification =
+ spark("SHOW CURRENT NAMESPACE", -35)
+ val SetCatalog: SqlStatementClassification = spark("SET CATALOG", -36)
+ val CreateMetricViewStmt: SqlStatementClassification = spark("CREATE METRIC
VIEW", -37)
+
+ private def spark(identifier: String, code: Int): SqlStatementClassification
= {
+ assert(code < 0, s"Spark statement codes must be negative, got $code")
+ SqlStatementClassification(statementIdentifier = identifier, statementCode
= code)
+ }
+
+ /** Classify an unresolved logical plan. */
+ def classify(plan: LogicalPlan): SqlStatementClassification = plan match {
+ case UnresolvedWith(child, _, _) => classify(child)
+ case _: CompoundBody => BeginEnd
+ case _: InsertIntoStatement => Insert
+ case _: DeleteFromTable | _: DeleteFromTableWithFilters => DeleteWhere
+ case _: UpdateTable => UpdateWhere
+ case _: MergeIntoTable => Merge
+ case _: CreateTableAsSelect | _: ReplaceTableAsSelect => CreateTable
+ case _: CreateTable | _: CreateTableLike | _: ReplaceTable => CreateTable
+ case _: CreateView | _: CreateViewCommand | _: CreateTempViewUsing =>
CreateView
+ case _: DropTable => DropTable
+ case _: DropView => DropView
+ case _: CreateNamespace => CreateSchema
+ case _: DropNamespace => DropSchema
+ case _: SetCatalogAndNamespace | _: SetNamespaceCommand => SetSchema
+ case _: SetCatalogCommand => SetCatalog
+ case _: TruncateTable => TruncateTable
+ case _: CreateFunction | _: CreateFunctionCommand |
+ _: CreateUserDefinedFunction | _: CreateUserDefinedFunctionCommand =>
+ CreateRoutine
+ case _: DropFunction | _: DropFunctionCommand => DropRoutine
+ case _: UnresolvedExecuteImmediate => ExecuteImmediate
+ case _: Call => Call
+ case _: CommentOnTable => CommentOnTable
+ case _: AlterTableCommand | _: RenameTable => AlterTable
+ case _: CacheTable => CacheTable
+ case _: CacheTableAsSelect => CacheTableAsSelect
+ case _: UncacheTable => UncacheTable
+ case _: RefreshTable => RefreshTable
+ case _: ShowTables | _: ShowTablesExtended => ShowTables
+ case _: DescribeRelation | _: DescribeTablePartition | _: DescribeColumn =>
+ DescribeTable
+ case _: DescribeQueryCommand => DescribeQuery
+ case _: AnalyzeTable | _: AnalyzeTables | _: AnalyzeColumn => AnalyzeTable
+ case _: CreateVariable => DeclareVariable
+ case _: SetVariable => SetVariable
+ case _: DropVariable => DropVariable
+ case _: ShowTableProperties => ShowTableProperties
+ case _: DescribeNamespace => DescribeNamespace
+ case _: ShowFunctions => ShowFunctions
+ case _: DescribeFunction => DescribeFunction
+ case _: ShowCreateTable => ShowCreateTable
+ case _: ShowColumns => ShowColumns
+ case _: ShowPartitions | _: ShowTablePartition => ShowPartitions
+ case _: ShowViews => ShowViews
+ case _: RefreshFunction => RefreshFunction
+ case _: CommentOnNamespace => CommentOnNamespace
+ case _: ExplainCommand => Explain
+ case _: SetCommand => Set
+ case _: ResetCommand => Reset
+ case _: AddJarsCommand => AddJar
+ case _: AddFilesCommand => AddFile
+ case _: AddArchivesCommand => AddArchive
+ case _: ListJarsCommand => ListJar
+ case _: ListFilesCommand => ListFile
+ case ClearCacheCommand => ClearCache
+ case _: RefreshResource => RefreshResourceCmd
+ case _: ShowCatalogsCommand => ShowCatalogs
+ case _: ShowCurrentNamespaceCommand => ShowCurrentNamespace
+ case _: CreateMetricView | _: CreateMetricViewCommand =>
CreateMetricViewStmt
+ case _: Command => Unrecognized
+ case p if isQueryPlan(p) => Select
+ case _ => Unrecognized
+ }
+
+ /**
+ * Allowlisted query-shaped plans. Unknown non-command plans are not assumed
+ * to be SELECT.
+ */
+ private def isQueryPlan(plan: LogicalPlan): Boolean = plan match {
+ case _: Project | _: Aggregate | _: Distinct | _: Filter | _: Sort |
+ _: GlobalLimit | _: LocalLimit | _: Join | _: Union | _: Except |
+ _: Intersect | _: SubqueryAlias | _: Repartition |
+ _: RepartitionByExpression | _: Sample | _: Range |
+ _: OneRowRelation | _: LocalRelation | _: Deduplicate |
+ _: Expand | _: Generate | _: Window | _: Tail | _: Offset |
+ _: LateralJoin | _: UnresolvedHaving | _: CollectMetrics |
+ _: WithCTE | _: UnresolvedRelation | _: UnresolvedInlineTable |
+ _: ResolvedInlineTable | _: RelationTimeTravel |
+ _: UnresolvedTableValuedFunction => true
+ case _ => false
+ }
+}
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala
b/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala
index 52c6821d0001..1ee7699fff42 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala
@@ -22,7 +22,7 @@ import org.apache.spark.sql.artifact.ArtifactManager
import org.apache.spark.sql.catalyst.analysis.{Analyzer,
EvalSubqueriesForTimeTravel, FunctionRegistry, InvokeProcedures,
ReplaceCharWithVarchar, ResolveDataSource, ResolveEventTimeWatermark,
ResolveExecuteImmediate, ResolveMetricView, ResolveSessionCatalog,
ResolveSetCatalogCommand, ResolveTranspose, TableFunctionRegistry}
import org.apache.spark.sql.catalyst.analysis.resolver.ResolverExtension
import org.apache.spark.sql.catalyst.catalog.{FunctionExpressionBuilder,
SessionCatalog}
-import org.apache.spark.sql.catalyst.expressions.{Expression,
ExtractSemiStructuredFields}
+import org.apache.spark.sql.catalyst.expressions.{Expression,
ExtractSemiStructuredFields, ParseSql}
import org.apache.spark.sql.catalyst.normalizer.NormalizeCTEIds
import org.apache.spark.sql.catalyst.optimizer.Optimizer
import org.apache.spark.sql.catalyst.parser.ParserInterface
@@ -96,7 +96,12 @@ abstract class BaseSessionStateBuilder(
*/
protected lazy val functionRegistry: FunctionRegistry = {
parentState.map(_.functionRegistry.clone())
-
.getOrElse(extensions.registerFunctions(FunctionRegistry.builtin.clone()))
+ .getOrElse {
+ val registry = FunctionRegistry.builtin.clone()
+ // sql/core-only builtins that need SparkSqlParser.
+ ParseSql.register(registry)
+ extensions.registerFunctions(registry)
+ }
}
/**
diff --git a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
index 7c389ed360ab..8a4d226fd94a 100644
--- a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
+++ b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
@@ -274,6 +274,7 @@
| org.apache.spark.sql.catalyst.expressions.OctetLength | octet_length |
SELECT octet_length('Spark SQL') | struct<octet_length(Spark SQL):int> |
| org.apache.spark.sql.catalyst.expressions.Or | or | SELECT true or false |
struct<(true OR false):boolean> |
| org.apache.spark.sql.catalyst.expressions.Overlay | overlay | SELECT
overlay('Spark SQL' PLACING '_' FROM 6) | struct<overlay(Spark SQL, _, 6,
-1):string> |
+| org.apache.spark.sql.catalyst.expressions.ParseSql | parse_sql | SELECT
parse_sql('SELECT a, b FROM t') | struct<parse_sql(SELECT a, b FROM t):string> |
| org.apache.spark.sql.catalyst.expressions.ParseToDate | to_date | SELECT
to_date('2009-07-30 04:17:52') | struct<to_date(2009-07-30 04:17:52):date> |
| org.apache.spark.sql.catalyst.expressions.ParseToTimestamp | to_timestamp |
SELECT to_timestamp('2016-12-31 00:12:00') | struct<to_timestamp(2016-12-31
00:12:00):timestamp> |
|
org.apache.spark.sql.catalyst.expressions.ParseToTimestampLTZExpressionBuilder
| to_timestamp_ltz | SELECT to_timestamp_ltz('2016-12-31 00:12:00') |
struct<to_timestamp_ltz(2016-12-31 00:12:00):timestamp> |
diff --git
a/sql/core/src/test/resources/sql-tests/analyzer-results/parse-sql-gating.sql.out
b/sql/core/src/test/resources/sql-tests/analyzer-results/parse-sql-gating.sql.out
new file mode 100644
index 000000000000..64ac040ca2e8
--- /dev/null
+++
b/sql/core/src/test/resources/sql-tests/analyzer-results/parse-sql-gating.sql.out
@@ -0,0 +1,14 @@
+-- Automatically generated by SQLQueryTestSuite
+-- !query
+SELECT parse_sql('SELECT 1')
+-- !query analysis
+org.apache.spark.sql.AnalysisException
+{
+ "errorClass" : "FEATURE_NOT_ENABLED",
+ "sqlState" : "56038",
+ "messageParameters" : {
+ "configKey" : "spark.sql.function.parseSql.enabled",
+ "configValue" : "true",
+ "featureName" : "parse_sql"
+ }
+}
diff --git
a/sql/core/src/test/resources/sql-tests/analyzer-results/parse-sql.sql.out
b/sql/core/src/test/resources/sql-tests/analyzer-results/parse-sql.sql.out
new file mode 100644
index 000000000000..bc9bbba7c3c7
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/analyzer-results/parse-sql.sql.out
@@ -0,0 +1,599 @@
+-- Automatically generated by SQLQueryTestSuite
+-- !query
+SELECT parse_sql(NULL)
+-- !query analysis
+Project [parse_sql(cast(null as string)) AS parse_sql(NULL)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT parse_sql('SELECT a, b FROM t')
+-- !query analysis
+Project [parse_sql(SELECT a, b FROM t) AS parse_sql(SELECT a, b FROM t)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT parse_sql('SELECT db.my_func(a), count(b) FROM cat.ns.t1 JOIN t2')
+-- !query analysis
+Project [parse_sql(SELECT db.my_func(a), count(b) FROM cat.ns.t1 JOIN t2) AS
parse_sql(SELECT db.my_func(a), count(b) FROM cat.ns.t1 JOIN t2)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT
+ get_json_object(result, '$.statement_identifier') AS statement_identifier,
+ get_json_object(result, '$.source_table_references[0][0]') AS first_table,
+ get_json_object(result, '$.select_list[1].name[0]') AS second_column
+FROM (SELECT parse_sql('SELECT a, b FROM t') AS result)
+-- !query analysis
+Project [get_json_object(result#x, $.statement_identifier) AS
statement_identifier#x, get_json_object(result#x,
$.source_table_references[0][0]) AS first_table#x, get_json_object(result#x,
$.select_list[1].name[0]) AS second_column#x]
++- SubqueryAlias __auto_generated_subquery_name
+ +- Project [parse_sql(SELECT a, b FROM t) AS result#x]
+ +- OneRowRelation
+
+
+-- !query
+SELECT parse_sql('INSERT INTO t SELECT 1')
+-- !query analysis
+Project [parse_sql(INSERT INTO t SELECT 1) AS parse_sql(INSERT INTO t SELECT
1)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT parse_sql('DELETE FROM t WHERE a = 1')
+-- !query analysis
+Project [parse_sql(DELETE FROM t WHERE a = 1) AS parse_sql(DELETE FROM t WHERE
a = 1)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT parse_sql('UPDATE t SET a = 1 WHERE b = 2')
+-- !query analysis
+Project [parse_sql(UPDATE t SET a = 1 WHERE b = 2) AS parse_sql(UPDATE t SET a
= 1 WHERE b = 2)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT parse_sql('MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN
DELETE')
+-- !query analysis
+Project [parse_sql(MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN
DELETE) AS parse_sql(MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN
DELETE)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT parse_sql('CREATE TABLE t (a INT)')
+-- !query analysis
+Project [parse_sql(CREATE TABLE t (a INT)) AS parse_sql(CREATE TABLE t (a
INT))#x]
++- OneRowRelation
+
+
+-- !query
+SELECT parse_sql('CREATE TABLE t AS SELECT 1 AS a')
+-- !query analysis
+Project [parse_sql(CREATE TABLE t AS SELECT 1 AS a) AS parse_sql(CREATE TABLE
t AS SELECT 1 AS a)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT parse_sql('DROP TABLE t')
+-- !query analysis
+Project [parse_sql(DROP TABLE t) AS parse_sql(DROP TABLE t)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT parse_sql('CACHE TABLE t')
+-- !query analysis
+Project [parse_sql(CACHE TABLE t) AS parse_sql(CACHE TABLE t)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT parse_sql('TABLE t')
+-- !query analysis
+Project [parse_sql(TABLE t) AS parse_sql(TABLE t)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT parse_sql('VALUES (1), (2)')
+-- !query analysis
+Project [parse_sql(VALUES (1), (2)) AS parse_sql(VALUES (1), (2))#x]
++- OneRowRelation
+
+
+-- !query
+SELECT parse_sql('CREATE FUNCTION f AS ''x'' USING JAR ''y.jar''')
+-- !query analysis
+Project [parse_sql(CREATE FUNCTION f AS 'x' USING JAR 'y.jar') AS
parse_sql(CREATE FUNCTION f AS 'x' USING JAR 'y.jar')#x]
++- OneRowRelation
+
+
+-- !query
+SELECT parse_sql('DECLARE VARIABLE x INT')
+-- !query analysis
+Project [parse_sql(DECLARE VARIABLE x INT) AS parse_sql(DECLARE VARIABLE x
INT)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT parse_sql('SELECT * FROM t WHERE a = :foo AND b = ?')
+-- !query analysis
+Project [parse_sql(SELECT * FROM t WHERE a = :foo AND b = ?) AS
parse_sql(SELECT * FROM t WHERE a = :foo AND b = ?)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT parse_sql('WITH cte AS (SELECT a FROM hidden_base) SELECT a FROM cte')
+-- !query analysis
+Project [parse_sql(WITH cte AS (SELECT a FROM hidden_base) SELECT a FROM cte)
AS parse_sql(WITH cte AS (SELECT a FROM hidden_base) SELECT a FROM cte)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT parse_sql('SELECT * FROM real_t WHERE EXISTS (WITH real_t AS (SELECT *
FROM inner_base) SELECT * FROM real_t)')
+-- !query analysis
+Project [parse_sql(SELECT * FROM real_t WHERE EXISTS (WITH real_t AS (SELECT *
FROM inner_base) SELECT * FROM real_t)) AS parse_sql(SELECT * FROM real_t WHERE
EXISTS (WITH real_t AS (SELECT * FROM inner_base) SELECT * FROM real_t))#x]
++- OneRowRelation
+
+
+-- !query
+SELECT parse_sql('WITH a AS (SELECT * FROM b), b AS (SELECT 1 AS x) SELECT *
FROM a')
+-- !query analysis
+Project [parse_sql(WITH a AS (SELECT * FROM b), b AS (SELECT 1 AS x) SELECT *
FROM a) AS parse_sql(WITH a AS (SELECT * FROM b), b AS (SELECT 1 AS x) SELECT *
FROM a)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT parse_sql('SELECT (SELECT max(v) FROM scalar_src) AS m, t.a FROM
outer_t t WHERE EXISTS (SELECT 1 FROM exists_src e WHERE e.id = t.id)')
+-- !query analysis
+Project [parse_sql(SELECT (SELECT max(v) FROM scalar_src) AS m, t.a FROM
outer_t t WHERE EXISTS (SELECT 1 FROM exists_src e WHERE e.id = t.id)) AS
parse_sql(SELECT (SELECT max(v) FROM scalar_src) AS m, t.a FROM outer_t t WHERE
EXISTS (SELECT 1 FROM exists_src e WHERE e.id = t.id))#x]
++- OneRowRelation
+
+
+-- !query
+SELECT parse_sql(
+'SELECT coalesce(t.a, 0), sum(abs(t.b)) OVER (
+ PARTITION BY lower(t.c) ORDER BY length(t.d))
+ FROM left_t t
+ JOIN right_t r ON hash(t.id) = hash(r.id)
+ JOIN LATERAL range(cast(t.n AS BIGINT)) rng
+ WHERE startswith(t.c, ''x'')
+ AND EXISTS (SELECT max(s.v) FROM scalar_t s WHERE s.id = t.id)
+ GROUP BY coalesce(t.a, 0), t.b, t.c, t.d
+ HAVING count_if(t.b > 0) > 0
+ ORDER BY greatest(t.a, 1)')
+-- !query analysis
+Project [parse_sql(SELECT coalesce(t.a, 0), sum(abs(t.b)) OVER (
+ PARTITION BY lower(t.c) ORDER BY length(t.d))
+ FROM left_t t
+ JOIN right_t r ON hash(t.id) = hash(r.id)
+ JOIN LATERAL range(cast(t.n AS BIGINT)) rng
+ WHERE startswith(t.c, 'x')
+ AND EXISTS (SELECT max(s.v) FROM scalar_t s WHERE s.id = t.id)
+ GROUP BY coalesce(t.a, 0), t.b, t.c, t.d
+ HAVING count_if(t.b > 0) > 0
+ ORDER BY greatest(t.a, 1)) AS parse_sql(SELECT coalesce(t.a, 0),
sum(abs(t.b)) OVER (
+ PARTITION BY lower(t.c) ORDER BY length(t.d))
+ FROM left_t t
+ JOIN right_t r ON hash(t.id) = hash(r.id)
+ JOIN LATERAL range(cast(t.n AS BIGINT)) rng
+ WHERE startswith(t.c, 'x')
+ AND EXISTS (SELECT max(s.v) FROM scalar_t s WHERE s.id = t.id)
+ GROUP BY coalesce(t.a, 0), t.b, t.c, t.d
+ HAVING count_if(t.b > 0) > 0
+ ORDER BY greatest(t.a, 1))#x]
++- OneRowRelation
+
+
+-- !query
+SELECT parse_sql(
+'MERGE INTO target t
+ USING (
+ SELECT id, normalize_name(name) AS name
+ FROM source
+ WHERE is_valid(id)
+ ) s
+ ON hash(t.id) = hash(s.id)
+ WHEN MATCHED AND should_update(t.name, s.name) THEN
+ UPDATE SET name = coalesce(s.name, upper(t.name))
+ WHEN NOT MATCHED THEN
+ INSERT (id, name) VALUES (s.id, lower(s.name))')
+-- !query analysis
+Project [parse_sql(MERGE INTO target t
+ USING (
+ SELECT id, normalize_name(name) AS name
+ FROM source
+ WHERE is_valid(id)
+ ) s
+ ON hash(t.id) = hash(s.id)
+ WHEN MATCHED AND should_update(t.name, s.name) THEN
+ UPDATE SET name = coalesce(s.name, upper(t.name))
+ WHEN NOT MATCHED THEN
+ INSERT (id, name) VALUES (s.id, lower(s.name))) AS parse_sql(MERGE INTO
target t
+ USING (
+ SELECT id, normalize_name(name) AS name
+ FROM source
+ WHERE is_valid(id)
+ ) s
+ ON hash(t.id) = hash(s.id)
+ WHEN MATCHED AND should_update(t.name, s.name) THEN
+ UPDATE SET name = coalesce(s.name, upper(t.name))
+ WHEN NOT MATCHED THEN
+ INSERT (id, name) VALUES (s.id, lower(s.name)))#x]
++- OneRowRelation
+
+
+-- !query
+SELECT parse_sql(
+'CREATE TABLE defaults (
+ created DATE DEFAULT current_date(),
+ normalized STRING DEFAULT upper(''x'')
+ )')
+-- !query analysis
+Project [parse_sql(CREATE TABLE defaults (
+ created DATE DEFAULT current_date(),
+ normalized STRING DEFAULT upper('x')
+ )) AS parse_sql(CREATE TABLE defaults (
+ created DATE DEFAULT current_date(),
+ normalized STRING DEFAULT upper('x')
+ ))#x]
++- OneRowRelation
+
+
+-- !query
+SELECT parse_sql('SELEC FROM t')
+-- !query analysis
+Project [parse_sql(SELEC FROM t) AS parse_sql(SELEC FROM t)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT
+ get_json_object(result, '$.parse_success') AS parse_success,
+ get_json_object(result, '$.error.errorClass') AS error_class,
+ get_json_object(result, '$.error.queryContext[0].fragment') AS fragment
+FROM (SELECT parse_sql('SELEC FROM t') AS result)
+-- !query analysis
+Project [get_json_object(result#x, $.parse_success) AS parse_success#x,
get_json_object(result#x, $.error.errorClass) AS error_class#x,
get_json_object(result#x, $.error.queryContext[0].fragment) AS fragment#x]
++- SubqueryAlias __auto_generated_subquery_name
+ +- Project [parse_sql(SELEC FROM t) AS result#x]
+ +- OneRowRelation
+
+
+-- !query
+SELECT parse_sql(
+'SELECT *
+ FROM t
+ ORDER BY a
+ CLUSTER BY b')
+-- !query analysis
+Project [parse_sql(SELECT *
+ FROM t
+ ORDER BY a
+ CLUSTER BY b) AS parse_sql(SELECT *
+ FROM t
+ ORDER BY a
+ CLUSTER BY b)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT
+ get_json_object(result, '$.error.errorClass') AS error_class,
+ get_json_object(result, '$.error.line') AS line,
+ get_json_object(result, '$.error.position') AS position,
+ get_json_object(result, '$.error.queryContext[0].startIndex') AS start_index
+FROM (
+ SELECT parse_sql(
+'SELECT *
+ FROM t
+ ORDER BY a
+ CLUSTER BY b') AS result
+)
+-- !query analysis
+Project [get_json_object(result#x, $.error.errorClass) AS error_class#x,
get_json_object(result#x, $.error.line) AS line#x, get_json_object(result#x,
$.error.position) AS position#x, get_json_object(result#x,
$.error.queryContext[0].startIndex) AS start_index#x]
++- SubqueryAlias __auto_generated_subquery_name
+ +- Project [parse_sql(SELECT *
+ FROM t
+ ORDER BY a
+ CLUSTER BY b) AS result#x]
+ +- OneRowRelation
+
+
+-- !query
+SELECT parse_sql('')
+-- !query analysis
+Project [parse_sql() AS parse_sql()#x]
++- OneRowRelation
+
+
+-- !query
+SELECT parse_sql('USE bad-name')
+-- !query analysis
+Project [parse_sql(USE bad-name) AS parse_sql(USE bad-name)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT parse_sql('WITH c AS (SELECT 1), c AS (SELECT 2) SELECT * FROM c')
+-- !query analysis
+Project [parse_sql(WITH c AS (SELECT 1), c AS (SELECT 2) SELECT * FROM c) AS
parse_sql(WITH c AS (SELECT 1), c AS (SELECT 2) SELECT * FROM c)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT parse_sql('MERGE INTO target USING source ON target.id = source.id')
+-- !query analysis
+Project [parse_sql(MERGE INTO target USING source ON target.id = source.id) AS
parse_sql(MERGE INTO target USING source ON target.id = source.id)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT parse_sql('EXPLAIN SELECT 1')
+-- !query analysis
+Project [parse_sql(EXPLAIN SELECT 1) AS parse_sql(EXPLAIN SELECT 1)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT parse_sql('SET spark.sql.adaptive.enabled=true')
+-- !query analysis
+Project [parse_sql(SET spark.sql.adaptive.enabled=true) AS parse_sql(SET
spark.sql.adaptive.enabled=true)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT parse_sql('ADD JAR /tmp/x.jar')
+-- !query analysis
+Project [parse_sql(ADD JAR /tmp/x.jar) AS parse_sql(ADD JAR /tmp/x.jar)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT parse_sql('CREATE VIEW v AS SELECT a, b FROM t')
+-- !query analysis
+Project [parse_sql(CREATE VIEW v AS SELECT a, b FROM t) AS parse_sql(CREATE
VIEW v AS SELECT a, b FROM t)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT parse_sql('SELECT 1 AS IDENTIFIER(''alias.field'')')
+-- !query analysis
+Project [parse_sql(SELECT 1 AS IDENTIFIER('alias.field')) AS parse_sql(SELECT
1 AS IDENTIFIER('alias.field'))#x]
++- OneRowRelation
+
+
+-- !query
+SELECT parse_sql('SELECT DATE ''not-a-date''')
+-- !query analysis
+Project [parse_sql(SELECT DATE 'not-a-date') AS parse_sql(SELECT DATE
'not-a-date')#x]
++- OneRowRelation
+
+
+-- !query
+SELECT parse_sql(
+'BEGIN
+ SELECT 1;
+ SELEC 2;
+ END')
+-- !query analysis
+Project [parse_sql(BEGIN
+ SELECT 1;
+ SELEC 2;
+ END) AS parse_sql(BEGIN
+ SELECT 1;
+ SELEC 2;
+ END)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT
+ get_json_object(result, '$.error.errorClass') AS error_class,
+ get_json_object(result, '$.error.line') AS line,
+ get_json_object(result, '$.error.position') AS position,
+ get_json_object(result, '$.error.queryContext[0].fragment') AS fragment
+FROM (
+ SELECT parse_sql(
+'BEGIN
+ SELECT 1;
+ SELEC 2;
+ END') AS result
+)
+-- !query analysis
+Project [get_json_object(result#x, $.error.errorClass) AS error_class#x,
get_json_object(result#x, $.error.line) AS line#x, get_json_object(result#x,
$.error.position) AS position#x, get_json_object(result#x,
$.error.queryContext[0].fragment) AS fragment#x]
++- SubqueryAlias __auto_generated_subquery_name
+ +- Project [parse_sql(BEGIN
+ SELECT 1;
+ SELEC 2;
+ END) AS result#x]
+ +- OneRowRelation
+
+
+-- !query
+SELECT parse_sql(
+'BEGIN
+ lbl_begin: BEGIN
+ SELECT 1;
+ END lbl_end;
+ END')
+-- !query analysis
+Project [parse_sql(BEGIN
+ lbl_begin: BEGIN
+ SELECT 1;
+ END lbl_end;
+ END) AS parse_sql(BEGIN
+ lbl_begin: BEGIN
+ SELECT 1;
+ END lbl_end;
+ END)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT
+ get_json_object(result, '$.error.errorClass') AS error_class,
+ get_json_object(result, '$.error.line') AS line,
+ get_json_object(result, '$.error.position') AS position,
+ get_json_object(result, '$.error.queryContext[0].fragment') AS fragment
+FROM (
+ SELECT parse_sql(
+'BEGIN
+ lbl_begin: BEGIN
+ SELECT 1;
+ END lbl_end;
+ END') AS result
+)
+-- !query analysis
+Project [get_json_object(result#x, $.error.errorClass) AS error_class#x,
get_json_object(result#x, $.error.line) AS line#x, get_json_object(result#x,
$.error.position) AS position#x, get_json_object(result#x,
$.error.queryContext[0].fragment) AS fragment#x]
++- SubqueryAlias __auto_generated_subquery_name
+ +- Project [parse_sql(BEGIN
+ lbl_begin: BEGIN
+ SELECT 1;
+ END lbl_end;
+ END) AS result#x]
+ +- OneRowRelation
+
+
+-- !query
+SELECT sql_text, parse_sql(sql_text) FROM VALUES
+ ('SELECT 1'),
+ ('INSERT INTO t SELECT 1'),
+ ('CACHE TABLE t')
+AS t(sql_text)
+-- !query analysis
+Project [sql_text#x, parse_sql(sql_text#x) AS parse_sql(sql_text)#x]
++- SubqueryAlias t
+ +- LocalRelation [sql_text#x]
+
+
+-- !query
+SELECT parse_sql('BEGIN SELECT 1; END')
+-- !query analysis
+Project [parse_sql(BEGIN SELECT 1; END) AS parse_sql(BEGIN SELECT 1; END)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT parse_sql('BEGIN SELECT count(a) FROM script_t WHERE c = :p; END')
+-- !query analysis
+Project [parse_sql(BEGIN SELECT count(a) FROM script_t WHERE c = :p; END) AS
parse_sql(BEGIN SELECT count(a) FROM script_t WHERE c = :p; END)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT parse_sql('BEGIN SELECT * FROM t WHERE a = ?; END')
+-- !query analysis
+Project [parse_sql(BEGIN SELECT * FROM t WHERE a = ?; END) AS parse_sql(BEGIN
SELECT * FROM t WHERE a = ?; END)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT parse_sql('BEGIN IF (SELECT flag FROM gate) THEN INSERT INTO dest
SELECT * FROM src_if; ELSE DELETE FROM src_else; END IF; END')
+-- !query analysis
+Project [parse_sql(BEGIN IF (SELECT flag FROM gate) THEN INSERT INTO dest
SELECT * FROM src_if; ELSE DELETE FROM src_else; END IF; END) AS
parse_sql(BEGIN IF (SELECT flag FROM gate) THEN INSERT INTO dest SELECT * FROM
src_if; ELSE DELETE FROM src_else; END IF; END)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT parse_sql('BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN INSERT
INTO err_log SELECT * FROM failing_row; END; SELECT a FROM main_t; END')
+-- !query analysis
+Project [parse_sql(BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN INSERT
INTO err_log SELECT * FROM failing_row; END; SELECT a FROM main_t; END) AS
parse_sql(BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN INSERT INTO err_log
SELECT * FROM failing_row; END; SELECT a FROM main_t; END)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT parse_sql(
+'BEGIN
+ DECLARE EXIT HANDLER FOR SQLEXCEPTION
+ BEGIN
+ INSERT INTO error_log
+ SELECT format_string(''%s'', message) FROM error_source;
+ END;
+
+ WITH prepared AS (
+ SELECT id, normalize_name(name) AS name
+ FROM input_names
+ WHERE is_valid(id)
+ )
+ INSERT INTO output_names
+ SELECT id, upper(name) FROM prepared;
+
+ IF EXISTS (SELECT 1 FROM control_flags WHERE enabled()) THEN
+ UPDATE update_target
+ SET value = coalesce((SELECT max(value) FROM update_source), 0)
+ WHERE should_update(id);
+ ELSE
+ DELETE FROM delete_target
+ WHERE id IN (SELECT id FROM delete_source WHERE expired(ts));
+ END IF;
+
+ FOR row AS
+ SELECT id FROM loop_source WHERE ready(id)
+ DO
+ SELECT audit(row.id), count(*) FROM loop_body;
+ END FOR;
+ END')
+-- !query analysis
+Project [parse_sql(BEGIN
+ DECLARE EXIT HANDLER FOR SQLEXCEPTION
+ BEGIN
+ INSERT INTO error_log
+ SELECT format_string('%s', message) FROM error_source;
+ END;
+
+ WITH prepared AS (
+ SELECT id, normalize_name(name) AS name
+ FROM input_names
+ WHERE is_valid(id)
+ )
+ INSERT INTO output_names
+ SELECT id, upper(name) FROM prepared;
+
+ IF EXISTS (SELECT 1 FROM control_flags WHERE enabled()) THEN
+ UPDATE update_target
+ SET value = coalesce((SELECT max(value) FROM update_source), 0)
+ WHERE should_update(id);
+ ELSE
+ DELETE FROM delete_target
+ WHERE id IN (SELECT id FROM delete_source WHERE expired(ts));
+ END IF;
+
+ FOR row AS
+ SELECT id FROM loop_source WHERE ready(id)
+ DO
+ SELECT audit(row.id), count(*) FROM loop_body;
+ END FOR;
+ END) AS parse_sql(BEGIN
+ DECLARE EXIT HANDLER FOR SQLEXCEPTION
+ BEGIN
+ INSERT INTO error_log
+ SELECT format_string('%s', message) FROM error_source;
+ END;
+
+ WITH prepared AS (
+ SELECT id, normalize_name(name) AS name
+ FROM input_names
+ WHERE is_valid(id)
+ )
+ INSERT INTO output_names
+ SELECT id, upper(name) FROM prepared;
+
+ IF EXISTS (SELECT 1 FROM control_flags WHERE enabled()) THEN
+ UPDATE update_target
+ SET value = coalesce((SELECT max(value) FROM update_source), 0)
+ WHERE should_update(id);
+ ELSE
+ DELETE FROM delete_target
+ WHERE id IN (SELECT id FROM delete_source WHERE expired(ts));
+ END IF;
+
+ FOR row AS
+ SELECT id FROM loop_source WHERE ready(id)
+ DO
+ SELECT audit(row.id), count(*) FROM loop_body;
+ END FOR;
+ END)#x]
++- OneRowRelation
diff --git a/sql/core/src/test/resources/sql-tests/inputs/parse-sql-gating.sql
b/sql/core/src/test/resources/sql-tests/inputs/parse-sql-gating.sql
new file mode 100644
index 000000000000..ae2a650f0859
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/parse-sql-gating.sql
@@ -0,0 +1,4 @@
+-- parse_sql is off by default while the JSON contract is still evolving.
+--SET spark.sql.function.parseSql.enabled=false
+
+SELECT parse_sql('SELECT 1');
diff --git a/sql/core/src/test/resources/sql-tests/inputs/parse-sql.sql
b/sql/core/src/test/resources/sql-tests/inputs/parse-sql.sql
new file mode 100644
index 000000000000..b4eb15ffbc38
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/parse-sql.sql
@@ -0,0 +1,249 @@
+-- End-to-end coverage for parse_sql (SPARK-58738).
+-- Returns compact JSON for parse-only statement analysis via SparkSqlParser.
+-- Off by default while the JSON contract is still evolving.
+--SET spark.sql.function.parseSql.enabled=true
+
+-- null input
+SELECT parse_sql(NULL);
+
+-- basic SELECT classification and references
+SELECT parse_sql('SELECT a, b FROM t');
+SELECT parse_sql('SELECT db.my_func(a), count(b) FROM cat.ns.t1 JOIN t2');
+
+-- JSON-path access over one shared successful parse result
+SELECT
+ get_json_object(result, '$.statement_identifier') AS statement_identifier,
+ get_json_object(result, '$.source_table_references[0][0]') AS first_table,
+ get_json_object(result, '$.select_list[1].name[0]') AS second_column
+FROM (SELECT parse_sql('SELECT a, b FROM t') AS result);
+
+-- DML
+SELECT parse_sql('INSERT INTO t SELECT 1');
+SELECT parse_sql('DELETE FROM t WHERE a = 1');
+SELECT parse_sql('UPDATE t SET a = 1 WHERE b = 2');
+SELECT parse_sql('MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN
DELETE');
+
+-- DDL / CTAS
+SELECT parse_sql('CREATE TABLE t (a INT)');
+SELECT parse_sql('CREATE TABLE t AS SELECT 1 AS a');
+SELECT parse_sql('DROP TABLE t');
+
+-- Spark-only statements (negative Table 39 codes)
+SELECT parse_sql('CACHE TABLE t');
+
+-- TABLE / VALUES are SELECT-shaped (not Unrecognized)
+SELECT parse_sql('TABLE t');
+SELECT parse_sql('VALUES (1), (2)');
+
+-- function / variable names are not target or source table references
+SELECT parse_sql('CREATE FUNCTION f AS ''x'' USING JAR ''y.jar''');
+SELECT parse_sql('DECLARE VARIABLE x INT');
+
+-- parameter markers
+SELECT parse_sql('SELECT * FROM t WHERE a = :foo AND b = ?');
+
+-- CTE: lineage excludes CTE names; still walks CTE bodies for real tables
+SELECT parse_sql('WITH cte AS (SELECT a FROM hidden_base) SELECT a FROM cte');
+
+-- CTE shadowing is scoped: the inner CTE real_t does not hide the outer table
+SELECT parse_sql('SELECT * FROM real_t WHERE EXISTS (WITH real_t AS (SELECT *
FROM inner_base) SELECT * FROM real_t)');
+
+-- a CTE definition sees only preceding aliases, so b below is the real table
+SELECT parse_sql('WITH a AS (SELECT * FROM b), b AS (SELECT 1 AS x) SELECT *
FROM a');
+
+-- nested subqueries
+SELECT parse_sql('SELECT (SELECT max(v) FROM scalar_src) AS m, t.a FROM
outer_t t WHERE EXISTS (SELECT 1 FROM exists_src e WHERE e.id = t.id)');
+
+-- functions in projection, window, join, TVF, predicates, subquery, grouping,
and ordering
+SELECT parse_sql(
+'SELECT coalesce(t.a, 0), sum(abs(t.b)) OVER (
+ PARTITION BY lower(t.c) ORDER BY length(t.d))
+ FROM left_t t
+ JOIN right_t r ON hash(t.id) = hash(r.id)
+ JOIN LATERAL range(cast(t.n AS BIGINT)) rng
+ WHERE startswith(t.c, ''x'')
+ AND EXISTS (SELECT max(s.v) FROM scalar_t s WHERE s.id = t.id)
+ GROUP BY coalesce(t.a, 0), t.b, t.c, t.d
+ HAVING count_if(t.b > 0) > 0
+ ORDER BY greatest(t.a, 1)');
+
+-- functions and tables throughout a multiline MERGE
+SELECT parse_sql(
+'MERGE INTO target t
+ USING (
+ SELECT id, normalize_name(name) AS name
+ FROM source
+ WHERE is_valid(id)
+ ) s
+ ON hash(t.id) = hash(s.id)
+ WHEN MATCHED AND should_update(t.name, s.name) THEN
+ UPDATE SET name = coalesce(s.name, upper(t.name))
+ WHEN NOT MATCHED THEN
+ INSERT (id, name) VALUES (s.id, lower(s.name))');
+
+-- functions embedded in DDL column defaults
+SELECT parse_sql(
+'CREATE TABLE defaults (
+ created DATE DEFAULT current_date(),
+ normalized STRING DEFAULT upper(''x'')
+ )');
+
+-- syntax error: dump the complete STANDARD error, including query context
+SELECT parse_sql('SELEC FROM t');
+
+-- JSON-path access over one shared parse result
+SELECT
+ get_json_object(result, '$.parse_success') AS parse_success,
+ get_json_object(result, '$.error.errorClass') AS error_class,
+ get_json_object(result, '$.error.queryContext[0].fragment') AS fragment
+FROM (SELECT parse_sql('SELEC FROM t') AS result);
+
+-- full multiline parse-time validation error, including context and location
+SELECT parse_sql(
+'SELECT *
+ FROM t
+ ORDER BY a
+ CLUSTER BY b');
+
+-- JSON-path access over one shared multiline parse result
+SELECT
+ get_json_object(result, '$.error.errorClass') AS error_class,
+ get_json_object(result, '$.error.line') AS line,
+ get_json_object(result, '$.error.position') AS position,
+ get_json_object(result, '$.error.queryContext[0].startIndex') AS start_index
+FROM (
+ SELECT parse_sql(
+'SELECT *
+ FROM t
+ ORDER BY a
+ CLUSTER BY b') AS result
+);
+
+-- parse-only validation errors beyond PARSE_SYNTAX_ERROR
+SELECT parse_sql('');
+SELECT parse_sql('USE bad-name');
+SELECT parse_sql('WITH c AS (SELECT 1), c AS (SELECT 2) SELECT * FROM c');
+SELECT parse_sql('MERGE INTO target USING source ON target.id = source.id');
+SELECT parse_sql('EXPLAIN SELECT 1');
+SELECT parse_sql('SET spark.sql.adaptive.enabled=true');
+SELECT parse_sql('ADD JAR /tmp/x.jar');
+SELECT parse_sql('CREATE VIEW v AS SELECT a, b FROM t');
+SELECT parse_sql('SELECT 1 AS IDENTIFIER(''alias.field'')');
+SELECT parse_sql('SELECT DATE ''not-a-date''');
+
+-- location for an error inside a multiline script
+--QUERY-DELIMITER-START
+SELECT parse_sql(
+'BEGIN
+ SELECT 1;
+ SELEC 2;
+ END');
+--QUERY-DELIMITER-END
+
+-- JSON-path access over one shared scripting parse result
+--QUERY-DELIMITER-START
+SELECT
+ get_json_object(result, '$.error.errorClass') AS error_class,
+ get_json_object(result, '$.error.line') AS line,
+ get_json_object(result, '$.error.position') AS position,
+ get_json_object(result, '$.error.queryContext[0].fragment') AS fragment
+FROM (
+ SELECT parse_sql(
+'BEGIN
+ SELECT 1;
+ SELEC 2;
+ END') AS result
+);
+--QUERY-DELIMITER-END
+
+-- location for a SQL scripting semantic validation error
+--QUERY-DELIMITER-START
+SELECT parse_sql(
+'BEGIN
+ lbl_begin: BEGIN
+ SELECT 1;
+ END lbl_end;
+ END');
+--QUERY-DELIMITER-END
+
+-- JSON-path access over one shared scripting validation result
+--QUERY-DELIMITER-START
+SELECT
+ get_json_object(result, '$.error.errorClass') AS error_class,
+ get_json_object(result, '$.error.line') AS line,
+ get_json_object(result, '$.error.position') AS position,
+ get_json_object(result, '$.error.queryContext[0].fragment') AS fragment
+FROM (
+ SELECT parse_sql(
+'BEGIN
+ lbl_begin: BEGIN
+ SELECT 1;
+ END lbl_end;
+ END') AS result
+);
+--QUERY-DELIMITER-END
+
+-- batch over a column of SQL text
+SELECT sql_text, parse_sql(sql_text) FROM VALUES
+ ('SELECT 1'),
+ ('INSERT INTO t SELECT 1'),
+ ('CACHE TABLE t')
+AS t(sql_text);
+
+-- BEGIN END scripts contain ';' inside the string literal; use query
delimiters
+-- so the test harness does not split on those semicolons.
+--QUERY-DELIMITER-START
+SELECT parse_sql('BEGIN SELECT 1; END');
+--QUERY-DELIMITER-END
+
+--QUERY-DELIMITER-START
+SELECT parse_sql('BEGIN SELECT count(a) FROM script_t WHERE c = :p; END');
+--QUERY-DELIMITER-END
+
+-- Positional markers under SingleStatement must not be double-counted.
+--QUERY-DELIMITER-START
+SELECT parse_sql('BEGIN SELECT * FROM t WHERE a = ?; END');
+--QUERY-DELIMITER-END
+
+--QUERY-DELIMITER-START
+SELECT parse_sql('BEGIN IF (SELECT flag FROM gate) THEN INSERT INTO dest
SELECT * FROM src_if; ELSE DELETE FROM src_else; END IF; END');
+--QUERY-DELIMITER-END
+
+--QUERY-DELIMITER-START
+SELECT parse_sql('BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN INSERT
INTO err_log SELECT * FROM failing_row; END; SELECT a FROM main_t; END');
+--QUERY-DELIMITER-END
+
+-- Complex, genuinely multiline script: dump the complete JSON result.
+--QUERY-DELIMITER-START
+SELECT parse_sql(
+'BEGIN
+ DECLARE EXIT HANDLER FOR SQLEXCEPTION
+ BEGIN
+ INSERT INTO error_log
+ SELECT format_string(''%s'', message) FROM error_source;
+ END;
+
+ WITH prepared AS (
+ SELECT id, normalize_name(name) AS name
+ FROM input_names
+ WHERE is_valid(id)
+ )
+ INSERT INTO output_names
+ SELECT id, upper(name) FROM prepared;
+
+ IF EXISTS (SELECT 1 FROM control_flags WHERE enabled()) THEN
+ UPDATE update_target
+ SET value = coalesce((SELECT max(value) FROM update_source), 0)
+ WHERE should_update(id);
+ ELSE
+ DELETE FROM delete_target
+ WHERE id IN (SELECT id FROM delete_source WHERE expired(ts));
+ END IF;
+
+ FOR row AS
+ SELECT id FROM loop_source WHERE ready(id)
+ DO
+ SELECT audit(row.id), count(*) FROM loop_body;
+ END FOR;
+ END');
+--QUERY-DELIMITER-END
diff --git
a/sql/core/src/test/resources/sql-tests/results/parse-sql-gating.sql.out
b/sql/core/src/test/resources/sql-tests/results/parse-sql-gating.sql.out
new file mode 100644
index 000000000000..aac028888704
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/parse-sql-gating.sql.out
@@ -0,0 +1,16 @@
+-- Automatically generated by SQLQueryTestSuite
+-- !query
+SELECT parse_sql('SELECT 1')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+{
+ "errorClass" : "FEATURE_NOT_ENABLED",
+ "sqlState" : "56038",
+ "messageParameters" : {
+ "configKey" : "spark.sql.function.parseSql.enabled",
+ "configValue" : "true",
+ "featureName" : "parse_sql"
+ }
+}
diff --git a/sql/core/src/test/resources/sql-tests/results/parse-sql.sql.out
b/sql/core/src/test/resources/sql-tests/results/parse-sql.sql.out
new file mode 100644
index 000000000000..986ce7c4ae94
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/parse-sql.sql.out
@@ -0,0 +1,571 @@
+-- Automatically generated by SQLQueryTestSuite
+-- !query
+SELECT parse_sql(NULL)
+-- !query schema
+struct<parse_sql(NULL):string>
+-- !query output
+NULL
+
+
+-- !query
+SELECT parse_sql('SELECT a, b FROM t')
+-- !query schema
+struct<parse_sql(SELECT a, b FROM t):string>
+-- !query output
+{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"source_table_references":[["t"]],"select_list":[{"name":["a"]},{"name":["b"]}]}
+
+
+-- !query
+SELECT parse_sql('SELECT db.my_func(a), count(b) FROM cat.ns.t1 JOIN t2')
+-- !query schema
+struct<parse_sql(SELECT db.my_func(a), count(b) FROM cat.ns.t1 JOIN t2):string>
+-- !query output
+{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"source_table_references":[["cat","ns","t1"],["t2"]],"function_references":[["db","my_func"],["count"]],"select_list":[{"name":[]},{"name":[]}]}
+
+
+-- !query
+SELECT
+ get_json_object(result, '$.statement_identifier') AS statement_identifier,
+ get_json_object(result, '$.source_table_references[0][0]') AS first_table,
+ get_json_object(result, '$.select_list[1].name[0]') AS second_column
+FROM (SELECT parse_sql('SELECT a, b FROM t') AS result)
+-- !query schema
+struct<statement_identifier:string,first_table:string,second_column:string>
+-- !query output
+SELECT t b
+
+
+-- !query
+SELECT parse_sql('INSERT INTO t SELECT 1')
+-- !query schema
+struct<parse_sql(INSERT INTO t SELECT 1):string>
+-- !query output
+{"parse_success":true,"statement_identifier":"INSERT","statement_code":50,"target_table_references":[["t"]],"select_list":[{"name":[]}]}
+
+
+-- !query
+SELECT parse_sql('DELETE FROM t WHERE a = 1')
+-- !query schema
+struct<parse_sql(DELETE FROM t WHERE a = 1):string>
+-- !query output
+{"parse_success":true,"statement_identifier":"DELETE
WHERE","statement_code":19,"target_table_references":[["t"]]}
+
+
+-- !query
+SELECT parse_sql('UPDATE t SET a = 1 WHERE b = 2')
+-- !query schema
+struct<parse_sql(UPDATE t SET a = 1 WHERE b = 2):string>
+-- !query output
+{"parse_success":true,"statement_identifier":"UPDATE
WHERE","statement_code":82,"target_table_references":[["t"]]}
+
+
+-- !query
+SELECT parse_sql('MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN
DELETE')
+-- !query schema
+struct<parse_sql(MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN
DELETE):string>
+-- !query output
+{"parse_success":true,"statement_identifier":"MERGE","statement_code":128,"target_table_references":[["t"]],"source_table_references":[["s"]]}
+
+
+-- !query
+SELECT parse_sql('CREATE TABLE t (a INT)')
+-- !query schema
+struct<parse_sql(CREATE TABLE t (a INT)):string>
+-- !query output
+{"parse_success":true,"statement_identifier":"CREATE
TABLE","statement_code":77,"target_table_references":[["t"]]}
+
+
+-- !query
+SELECT parse_sql('CREATE TABLE t AS SELECT 1 AS a')
+-- !query schema
+struct<parse_sql(CREATE TABLE t AS SELECT 1 AS a):string>
+-- !query output
+{"parse_success":true,"statement_identifier":"CREATE
TABLE","statement_code":77,"target_table_references":[["t"]],"select_list":[{"name":["a"]}]}
+
+
+-- !query
+SELECT parse_sql('DROP TABLE t')
+-- !query schema
+struct<parse_sql(DROP TABLE t):string>
+-- !query output
+{"parse_success":true,"statement_identifier":"DROP
TABLE","statement_code":32,"target_table_references":[["t"]]}
+
+
+-- !query
+SELECT parse_sql('CACHE TABLE t')
+-- !query schema
+struct<parse_sql(CACHE TABLE t):string>
+-- !query output
+{"parse_success":true,"statement_identifier":"CACHE
TABLE","statement_code":-1,"target_table_references":[["t"]]}
+
+
+-- !query
+SELECT parse_sql('TABLE t')
+-- !query schema
+struct<parse_sql(TABLE t):string>
+-- !query output
+{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"source_table_references":[["t"]]}
+
+
+-- !query
+SELECT parse_sql('VALUES (1), (2)')
+-- !query schema
+struct<parse_sql(VALUES (1), (2)):string>
+-- !query output
+{"parse_success":true,"statement_identifier":"SELECT","statement_code":21}
+
+
+-- !query
+SELECT parse_sql('CREATE FUNCTION f AS ''x'' USING JAR ''y.jar''')
+-- !query schema
+struct<parse_sql(CREATE FUNCTION f AS 'x' USING JAR 'y.jar'):string>
+-- !query output
+{"parse_success":true,"statement_identifier":"CREATE
ROUTINE","statement_code":14}
+
+
+-- !query
+SELECT parse_sql('DECLARE VARIABLE x INT')
+-- !query schema
+struct<parse_sql(DECLARE VARIABLE x INT):string>
+-- !query output
+{"parse_success":true,"statement_identifier":"DECLARE
VARIABLE","statement_code":-8}
+
+
+-- !query
+SELECT parse_sql('SELECT * FROM t WHERE a = :foo AND b = ?')
+-- !query schema
+struct<parse_sql(SELECT * FROM t WHERE a = :foo AND b = ?):string>
+-- !query output
+{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"source_table_references":[["t"]],"select_list":[{"name":["*"]}],"parameter_markers":{"named":["foo"],"unnamed_count":1}}
+
+
+-- !query
+SELECT parse_sql('WITH cte AS (SELECT a FROM hidden_base) SELECT a FROM cte')
+-- !query schema
+struct<parse_sql(WITH cte AS (SELECT a FROM hidden_base) SELECT a FROM
cte):string>
+-- !query output
+{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"source_table_references":[["hidden_base"]],"select_list":[{"name":["a"]}]}
+
+
+-- !query
+SELECT parse_sql('SELECT * FROM real_t WHERE EXISTS (WITH real_t AS (SELECT *
FROM inner_base) SELECT * FROM real_t)')
+-- !query schema
+struct<parse_sql(SELECT * FROM real_t WHERE EXISTS (WITH real_t AS (SELECT *
FROM inner_base) SELECT * FROM real_t)):string>
+-- !query output
+{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"source_table_references":[["inner_base"],["real_t"]],"select_list":[{"name":["*"]}]}
+
+
+-- !query
+SELECT parse_sql('WITH a AS (SELECT * FROM b), b AS (SELECT 1 AS x) SELECT *
FROM a')
+-- !query schema
+struct<parse_sql(WITH a AS (SELECT * FROM b), b AS (SELECT 1 AS x) SELECT *
FROM a):string>
+-- !query output
+{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"source_table_references":[["b"]],"select_list":[{"name":["*"]}]}
+
+
+-- !query
+SELECT parse_sql('SELECT (SELECT max(v) FROM scalar_src) AS m, t.a FROM
outer_t t WHERE EXISTS (SELECT 1 FROM exists_src e WHERE e.id = t.id)')
+-- !query schema
+struct<parse_sql(SELECT (SELECT max(v) FROM scalar_src) AS m, t.a FROM outer_t
t WHERE EXISTS (SELECT 1 FROM exists_src e WHERE e.id = t.id)):string>
+-- !query output
+{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"source_table_references":[["scalar_src"],["exists_src"],["outer_t"]],"function_references":[["max"]],"select_list":[{"name":["m"]},{"name":["t","a"]}]}
+
+
+-- !query
+SELECT parse_sql(
+'SELECT coalesce(t.a, 0), sum(abs(t.b)) OVER (
+ PARTITION BY lower(t.c) ORDER BY length(t.d))
+ FROM left_t t
+ JOIN right_t r ON hash(t.id) = hash(r.id)
+ JOIN LATERAL range(cast(t.n AS BIGINT)) rng
+ WHERE startswith(t.c, ''x'')
+ AND EXISTS (SELECT max(s.v) FROM scalar_t s WHERE s.id = t.id)
+ GROUP BY coalesce(t.a, 0), t.b, t.c, t.d
+ HAVING count_if(t.b > 0) > 0
+ ORDER BY greatest(t.a, 1)')
+-- !query schema
+struct<parse_sql(SELECT coalesce(t.a, 0), sum(abs(t.b)) OVER (
+ PARTITION BY lower(t.c) ORDER BY length(t.d))
+ FROM left_t t
+ JOIN right_t r ON hash(t.id) = hash(r.id)
+ JOIN LATERAL range(cast(t.n AS BIGINT)) rng
+ WHERE startswith(t.c, 'x')
+ AND EXISTS (SELECT max(s.v) FROM scalar_t s WHERE s.id = t.id)
+ GROUP BY coalesce(t.a, 0), t.b, t.c, t.d
+ HAVING count_if(t.b > 0) > 0
+ ORDER BY greatest(t.a, 1)):string>
+-- !query output
+{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"source_table_references":[["scalar_t"],["left_t"],["right_t"]],"function_references":[["greatest"],["count_if"],["coalesce"],["sum"],["abs"],["lower"],["length"],["startswith"],["max"],["range"],["hash"]],"select_list":[{"name":[]},{"name":[]}]}
+
+
+-- !query
+SELECT parse_sql(
+'MERGE INTO target t
+ USING (
+ SELECT id, normalize_name(name) AS name
+ FROM source
+ WHERE is_valid(id)
+ ) s
+ ON hash(t.id) = hash(s.id)
+ WHEN MATCHED AND should_update(t.name, s.name) THEN
+ UPDATE SET name = coalesce(s.name, upper(t.name))
+ WHEN NOT MATCHED THEN
+ INSERT (id, name) VALUES (s.id, lower(s.name))')
+-- !query schema
+struct<parse_sql(MERGE INTO target t
+ USING (
+ SELECT id, normalize_name(name) AS name
+ FROM source
+ WHERE is_valid(id)
+ ) s
+ ON hash(t.id) = hash(s.id)
+ WHEN MATCHED AND should_update(t.name, s.name) THEN
+ UPDATE SET name = coalesce(s.name, upper(t.name))
+ WHEN NOT MATCHED THEN
+ INSERT (id, name) VALUES (s.id, lower(s.name))):string>
+-- !query output
+{"parse_success":true,"statement_identifier":"MERGE","statement_code":128,"target_table_references":[["target"]],"source_table_references":[["source"]],"function_references":[["hash"],["should_update"],["coalesce"],["upper"],["lower"],["normalize_name"],["is_valid"]]}
+
+
+-- !query
+SELECT parse_sql(
+'CREATE TABLE defaults (
+ created DATE DEFAULT current_date(),
+ normalized STRING DEFAULT upper(''x'')
+ )')
+-- !query schema
+struct<parse_sql(CREATE TABLE defaults (
+ created DATE DEFAULT current_date(),
+ normalized STRING DEFAULT upper('x')
+ )):string>
+-- !query output
+{"parse_success":true,"statement_identifier":"CREATE
TABLE","statement_code":77,"target_table_references":[["defaults"]],"function_references":[["current_date"],["upper"]]}
+
+
+-- !query
+SELECT parse_sql('SELEC FROM t')
+-- !query schema
+struct<parse_sql(SELEC FROM t):string>
+-- !query output
+{"parse_success":false,"error":{"errorClass":"PARSE_SYNTAX_ERROR","messageTemplate":"Syntax
error at or near
<error><hint>.","sqlState":"42601","messageParameters":{"error":"'SELEC'","hint":""},"queryContext":[{"objectType":"","objectName":"","startIndex":1,"stopIndex":12,"fragment":"SELEC
FROM t"}],"line":1,"position":0}}
+
+
+-- !query
+SELECT
+ get_json_object(result, '$.parse_success') AS parse_success,
+ get_json_object(result, '$.error.errorClass') AS error_class,
+ get_json_object(result, '$.error.queryContext[0].fragment') AS fragment
+FROM (SELECT parse_sql('SELEC FROM t') AS result)
+-- !query schema
+struct<parse_success:string,error_class:string,fragment:string>
+-- !query output
+false PARSE_SYNTAX_ERROR SELEC FROM t
+
+
+-- !query
+SELECT parse_sql(
+'SELECT *
+ FROM t
+ ORDER BY a
+ CLUSTER BY b')
+-- !query schema
+struct<parse_sql(SELECT *
+ FROM t
+ ORDER BY a
+ CLUSTER BY b):string>
+-- !query output
+{"parse_success":false,"error":{"errorClass":"UNSUPPORTED_FEATURE.COMBINATION_QUERY_RESULT_CLAUSES","messageTemplate":"The
feature is not supported: Combination of ORDER BY/SORT BY/DISTRIBUTE
BY/CLUSTER
BY.","sqlState":"0A000","queryContext":[{"objectType":"","objectName":"","startIndex":19,"stopIndex":42,"fragment":"ORDER
BY a\n CLUSTER BY b"}],"line":3,"position":1}}
+
+
+-- !query
+SELECT
+ get_json_object(result, '$.error.errorClass') AS error_class,
+ get_json_object(result, '$.error.line') AS line,
+ get_json_object(result, '$.error.position') AS position,
+ get_json_object(result, '$.error.queryContext[0].startIndex') AS start_index
+FROM (
+ SELECT parse_sql(
+'SELECT *
+ FROM t
+ ORDER BY a
+ CLUSTER BY b') AS result
+)
+-- !query schema
+struct<error_class:string,line:string,position:string,start_index:string>
+-- !query output
+UNSUPPORTED_FEATURE.COMBINATION_QUERY_RESULT_CLAUSES 3 1 19
+
+
+-- !query
+SELECT parse_sql('')
+-- !query schema
+struct<parse_sql():string>
+-- !query output
+{"parse_success":false,"error":{"errorClass":"PARSE_EMPTY_STATEMENT","messageTemplate":"Syntax
error, unexpected empty statement.","sqlState":"42617","line":1,"position":0}}
+
+
+-- !query
+SELECT parse_sql('USE bad-name')
+-- !query schema
+struct<parse_sql(USE bad-name):string>
+-- !query output
+{"parse_success":false,"error":{"errorClass":"INVALID_IDENTIFIER","messageTemplate":"The
unquoted identifier <ident> is invalid and must be back quoted as:
`<ident>`.\nUnquoted identifiers can only contain ASCII letters ('a' - 'z', 'A'
- 'Z'), digits ('0' - '9'), and underbar ('_').\nUnquoted identifiers must also
not start with a digit.\nDifferent data sources and meta stores may impose
additional restrictions on valid
identifiers.","sqlState":"42602","messageParameters":{"ident":"bad-n [...]
+
+
+-- !query
+SELECT parse_sql('WITH c AS (SELECT 1), c AS (SELECT 2) SELECT * FROM c')
+-- !query schema
+struct<parse_sql(WITH c AS (SELECT 1), c AS (SELECT 2) SELECT * FROM c):string>
+-- !query output
+{"parse_success":false,"error":{"errorClass":"DUPLICATED_CTE_NAMES","messageTemplate":"CTE
definition can't have duplicate names:
<duplicateNames>.","sqlState":"42602","messageParameters":{"duplicateNames":"`c`"},"queryContext":[{"objectType":"","objectName":"","startIndex":1,"stopIndex":53,"fragment":"WITH
c AS (SELECT 1), c AS (SELECT 2) SELECT * FROM c"}],"line":1,"position":0}}
+
+
+-- !query
+SELECT parse_sql('MERGE INTO target USING source ON target.id = source.id')
+-- !query schema
+struct<parse_sql(MERGE INTO target USING source ON target.id =
source.id):string>
+-- !query output
+{"parse_success":false,"error":{"errorClass":"MERGE_WITHOUT_WHEN","messageTemplate":"There
must be at least one WHEN clause in a MERGE
statement.","sqlState":"42601","queryContext":[{"objectType":"","objectName":"","startIndex":1,"stopIndex":55,"fragment":"MERGE
INTO target USING source ON target.id = source.id"}],"line":1,"position":0}}
+
+
+-- !query
+SELECT parse_sql('EXPLAIN SELECT 1')
+-- !query schema
+struct<parse_sql(EXPLAIN SELECT 1):string>
+-- !query output
+{"parse_success":true,"statement_identifier":"EXPLAIN","statement_code":-23,"select_list":[{"name":[]}]}
+
+
+-- !query
+SELECT parse_sql('SET spark.sql.adaptive.enabled=true')
+-- !query schema
+struct<parse_sql(SET spark.sql.adaptive.enabled=true):string>
+-- !query output
+{"parse_success":true,"statement_identifier":"SET","statement_code":-24}
+
+
+-- !query
+SELECT parse_sql('ADD JAR /tmp/x.jar')
+-- !query schema
+struct<parse_sql(ADD JAR /tmp/x.jar):string>
+-- !query output
+{"parse_success":true,"statement_identifier":"ADD JAR","statement_code":-26}
+
+
+-- !query
+SELECT parse_sql('CREATE VIEW v AS SELECT a, b FROM t')
+-- !query schema
+struct<parse_sql(CREATE VIEW v AS SELECT a, b FROM t):string>
+-- !query output
+{"parse_success":true,"statement_identifier":"CREATE
VIEW","statement_code":84,"target_table_references":[["v"]],"source_table_references":[["t"]],"select_list":[{"name":["a"]},{"name":["b"]}]}
+
+
+-- !query
+SELECT parse_sql('SELECT 1 AS IDENTIFIER(''alias.field'')')
+-- !query schema
+struct<parse_sql(SELECT 1 AS IDENTIFIER('alias.field')):string>
+-- !query output
+{"parse_success":false,"error":{"errorClass":"IDENTIFIER_TOO_MANY_NAME_PARTS","messageTemplate":"<identifier>
is not a valid identifier as it has more than <limit> name
parts.","sqlState":"42601","messageParameters":{"identifier":"`alias`.`field`","limit":"1"},"queryContext":[{"objectType":"","objectName":"","startIndex":8,"stopIndex":37,"fragment":"1
AS IDENTIFIER('alias.field')"}],"line":1,"position":12}}
+
+
+-- !query
+SELECT parse_sql('SELECT DATE ''not-a-date''')
+-- !query schema
+struct<parse_sql(SELECT DATE 'not-a-date'):string>
+-- !query output
+{"parse_success":false,"error":{"errorClass":"INVALID_TYPED_LITERAL","messageTemplate":"The
value of the typed literal <valueType> is invalid:
<value>.","sqlState":"42604","messageParameters":{"value":"'not-a-date'","valueType":"\"DATE\""},"queryContext":[{"objectType":"","objectName":"","startIndex":8,"stopIndex":24,"fragment":"DATE
'not-a-date'"}],"line":1,"position":7}}
+
+
+-- !query
+SELECT parse_sql(
+'BEGIN
+ SELECT 1;
+ SELEC 2;
+ END')
+-- !query schema
+struct<parse_sql(BEGIN
+ SELECT 1;
+ SELEC 2;
+ END):string>
+-- !query output
+{"parse_success":false,"error":{"errorClass":"PARSE_SYNTAX_ERROR","messageTemplate":"Syntax
error at or near
<error><hint>.","sqlState":"42601","messageParameters":{"error":"'2'","hint":""},"queryContext":[{"objectType":"","objectName":"","startIndex":1,"stopIndex":35,"fragment":"BEGIN\n
SELECT 1;\n SELEC 2;\n END"}],"line":3,"position":9}}
+
+
+-- !query
+SELECT
+ get_json_object(result, '$.error.errorClass') AS error_class,
+ get_json_object(result, '$.error.line') AS line,
+ get_json_object(result, '$.error.position') AS position,
+ get_json_object(result, '$.error.queryContext[0].fragment') AS fragment
+FROM (
+ SELECT parse_sql(
+'BEGIN
+ SELECT 1;
+ SELEC 2;
+ END') AS result
+)
+-- !query schema
+struct<error_class:string,line:string,position:string,fragment:string>
+-- !query output
+PARSE_SYNTAX_ERROR 3 9 BEGIN
+ SELECT 1;
+ SELEC 2;
+ END
+
+
+-- !query
+SELECT parse_sql(
+'BEGIN
+ lbl_begin: BEGIN
+ SELECT 1;
+ END lbl_end;
+ END')
+-- !query schema
+struct<parse_sql(BEGIN
+ lbl_begin: BEGIN
+ SELECT 1;
+ END lbl_end;
+ END):string>
+-- !query output
+{"parse_success":false,"error":{"errorClass":"LABELS_MISMATCH","messageTemplate":"Begin
label <beginLabel> does not match the end label
<endLabel>.","sqlState":"42K0L","messageParameters":{"beginLabel":"`lbl_begin`","endLabel":"`lbl_end`"},"queryContext":[{"objectType":"","objectName":"","startIndex":10,"stopIndex":19,"fragment":"lbl_begin:"}],"line":2,"position":3}}
+
+
+-- !query
+SELECT
+ get_json_object(result, '$.error.errorClass') AS error_class,
+ get_json_object(result, '$.error.line') AS line,
+ get_json_object(result, '$.error.position') AS position,
+ get_json_object(result, '$.error.queryContext[0].fragment') AS fragment
+FROM (
+ SELECT parse_sql(
+'BEGIN
+ lbl_begin: BEGIN
+ SELECT 1;
+ END lbl_end;
+ END') AS result
+)
+-- !query schema
+struct<error_class:string,line:string,position:string,fragment:string>
+-- !query output
+LABELS_MISMATCH 2 3 lbl_begin:
+
+
+-- !query
+SELECT sql_text, parse_sql(sql_text) FROM VALUES
+ ('SELECT 1'),
+ ('INSERT INTO t SELECT 1'),
+ ('CACHE TABLE t')
+AS t(sql_text)
+-- !query schema
+struct<sql_text:string,parse_sql(sql_text):string>
+-- !query output
+CACHE TABLE t {"parse_success":true,"statement_identifier":"CACHE
TABLE","statement_code":-1,"target_table_references":[["t"]]}
+INSERT INTO t SELECT 1
{"parse_success":true,"statement_identifier":"INSERT","statement_code":50,"target_table_references":[["t"]],"select_list":[{"name":[]}]}
+SELECT 1
{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"select_list":[{"name":[]}]}
+
+
+-- !query
+SELECT parse_sql('BEGIN SELECT 1; END')
+-- !query schema
+struct<parse_sql(BEGIN SELECT 1; END):string>
+-- !query output
+{"parse_success":true,"statement_identifier":"BEGIN END","statement_code":-22}
+
+
+-- !query
+SELECT parse_sql('BEGIN SELECT count(a) FROM script_t WHERE c = :p; END')
+-- !query schema
+struct<parse_sql(BEGIN SELECT count(a) FROM script_t WHERE c = :p; END):string>
+-- !query output
+{"parse_success":true,"statement_identifier":"BEGIN
END","statement_code":-22,"source_table_references":[["script_t"]],"function_references":[["count"]],"parameter_markers":{"named":["p"]}}
+
+
+-- !query
+SELECT parse_sql('BEGIN SELECT * FROM t WHERE a = ?; END')
+-- !query schema
+struct<parse_sql(BEGIN SELECT * FROM t WHERE a = ?; END):string>
+-- !query output
+{"parse_success":true,"statement_identifier":"BEGIN
END","statement_code":-22,"source_table_references":[["t"]],"parameter_markers":{"unnamed_count":1}}
+
+
+-- !query
+SELECT parse_sql('BEGIN IF (SELECT flag FROM gate) THEN INSERT INTO dest
SELECT * FROM src_if; ELSE DELETE FROM src_else; END IF; END')
+-- !query schema
+struct<parse_sql(BEGIN IF (SELECT flag FROM gate) THEN INSERT INTO dest SELECT
* FROM src_if; ELSE DELETE FROM src_else; END IF; END):string>
+-- !query output
+{"parse_success":true,"statement_identifier":"BEGIN
END","statement_code":-22,"target_table_references":[["dest"],["src_else"]],"source_table_references":[["gate"],["src_if"]]}
+
+
+-- !query
+SELECT parse_sql('BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN INSERT
INTO err_log SELECT * FROM failing_row; END; SELECT a FROM main_t; END')
+-- !query schema
+struct<parse_sql(BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN INSERT INTO
err_log SELECT * FROM failing_row; END; SELECT a FROM main_t; END):string>
+-- !query output
+{"parse_success":true,"statement_identifier":"BEGIN
END","statement_code":-22,"target_table_references":[["err_log"]],"source_table_references":[["failing_row"],["main_t"]]}
+
+
+-- !query
+SELECT parse_sql(
+'BEGIN
+ DECLARE EXIT HANDLER FOR SQLEXCEPTION
+ BEGIN
+ INSERT INTO error_log
+ SELECT format_string(''%s'', message) FROM error_source;
+ END;
+
+ WITH prepared AS (
+ SELECT id, normalize_name(name) AS name
+ FROM input_names
+ WHERE is_valid(id)
+ )
+ INSERT INTO output_names
+ SELECT id, upper(name) FROM prepared;
+
+ IF EXISTS (SELECT 1 FROM control_flags WHERE enabled()) THEN
+ UPDATE update_target
+ SET value = coalesce((SELECT max(value) FROM update_source), 0)
+ WHERE should_update(id);
+ ELSE
+ DELETE FROM delete_target
+ WHERE id IN (SELECT id FROM delete_source WHERE expired(ts));
+ END IF;
+
+ FOR row AS
+ SELECT id FROM loop_source WHERE ready(id)
+ DO
+ SELECT audit(row.id), count(*) FROM loop_body;
+ END FOR;
+ END')
+-- !query schema
+struct<parse_sql(BEGIN
+ DECLARE EXIT HANDLER FOR SQLEXCEPTION
+ BEGIN
+ INSERT INTO error_log
+ SELECT format_string('%s', message) FROM error_source;
+ END;
+
+ WITH prepared AS (
+ SELECT id, normalize_name(name) AS name
+ FROM input_names
+ WHERE is_valid(id)
+ )
+ INSERT INTO output_names
+ SELECT id, upper(name) FROM prepared;
+
+ IF EXISTS (SELECT 1 FROM control_flags WHERE enabled()) THEN
+ UPDATE update_target
+ SET value = coalesce((SELECT max(value) FROM update_source), 0)
+ WHERE should_update(id);
+ ELSE
+ DELETE FROM delete_target
+ WHERE id IN (SELECT id FROM delete_source WHERE expired(ts));
+ END IF;
+
+ FOR row AS
+ SELECT id FROM loop_source WHERE ready(id)
+ DO
+ SELECT audit(row.id), count(*) FROM loop_body;
+ END FOR;
+ END):string>
+-- !query output
+{"parse_success":true,"statement_identifier":"BEGIN
END","statement_code":-22,"target_table_references":[["error_log"],["output_names"],["update_target"],["delete_target"]],"source_table_references":[["error_source"],["input_names"],["control_flags"],["update_source"],["delete_source"],["loop_source"],["loop_body"]],"function_references":[["format_string"],["normalize_name"],["is_valid"],["upper"],["enabled"],["coalesce"],["should_update"],["max"],["expired"],["ready"],["audit"],["count"]]}
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/ExpressionsSchemaSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/ExpressionsSchemaSuite.scala
index 4550ad76d43b..5e5a5bb8d580 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ExpressionsSchemaSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ExpressionsSchemaSuite.scala
@@ -23,6 +23,7 @@ import java.nio.file.Files
import scala.collection.mutable.ArrayBuffer
import org.apache.spark.sql.catalyst.util.stringToFile
+import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.test.SharedSparkSession
import org.apache.spark.tags.ExtendedSQLTest
import org.apache.spark.util.Utils
@@ -127,12 +128,15 @@ class ExpressionsSchemaSuite extends SharedSparkSession {
// AvroDataToCatalyst or CatalystDataToAvro classes which are
not available in this
// test.
case exampleRe(sql, _) =>
- val df = spark.sql(sql)
- val escapedSql = sql.replaceAll("\\|", "|")
- val schema = df.schema.catalogString.replaceAll("\\|", "|")
- val queryOutput = QueryOutput(className, funcName, escapedSql,
schema)
- outputBuffer += queryOutput.toString
- outputs += queryOutput
+ // parse_sql examples require the experimental feature flag.
+ withSQLConf(SQLConf.PARSE_SQL_ENABLED.key -> true.toString) {
+ val df = spark.sql(sql)
+ val escapedSql = sql.replaceAll("\\|", "|")
+ val schema = df.schema.catalogString.replaceAll("\\|", "|")
+ val queryOutput = QueryOutput(className, funcName, escapedSql,
schema)
+ outputBuffer += queryOutput.toString
+ outputs += queryOutput
+ }
case _ =>
}
}
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/catalyst/expressions/ParseSqlSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/catalyst/expressions/ParseSqlSuite.scala
new file mode 100644
index 000000000000..f44a409a0d64
--- /dev/null
+++
b/sql/core/src/test/scala/org/apache/spark/sql/catalyst/expressions/ParseSqlSuite.scala
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.expressions
+
+import org.json4s._
+import org.json4s.jackson.JsonMethods.parse
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
+import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
+import org.apache.spark.sql.catalyst.plans.SQLHelper
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.types.StringType
+import org.apache.spark.unsafe.types.UTF8String
+
+class ParseSqlSuite extends SparkFunSuite with ExpressionEvalHelper with
SQLHelper {
+
+ private def evalJson(sql: String): JValue = {
+ val result =
ParseSql(Literal(sql)).eval().asInstanceOf[UTF8String].toString
+ parse(result)
+ }
+
+ test("parse_sql is disabled by default") {
+ assert(!SQLConf.get.parseSqlEnabled)
+ checkError(
+ exception = intercept[AnalysisException] {
+ ParseSql(Literal("SELECT 1")).checkInputDataTypes()
+ },
+ condition = "FEATURE_NOT_ENABLED",
+ parameters = Map(
+ "featureName" -> "parse_sql",
+ "configKey" -> SQLConf.PARSE_SQL_ENABLED.key,
+ "configValue" -> "true"))
+ }
+
+ test("parse_sql type check succeeds when enabled") {
+ withSQLConf(SQLConf.PARSE_SQL_ENABLED.key -> "true") {
+ assert(ParseSql(Literal("SELECT 1")).checkInputDataTypes() ===
+ TypeCheckResult.TypeCheckSuccess)
+ }
+ }
+
+ test("parse_sql returns JSON for a valid SELECT") {
+ withSQLConf(SQLConf.PARSE_SQL_ENABLED.key -> "true") {
+ val j = evalJson("SELECT 1 AS a")
+ assert(j \ "parse_success" === JBool(true))
+ assert(j \ "statement_identifier" === JString("SELECT"))
+ assert(j \ "statement_code" === JInt(21))
+ }
+ }
+
+ test("parse_sql returns null for null input") {
+ withSQLConf(SQLConf.PARSE_SQL_ENABLED.key -> "true") {
+ checkEvaluation(ParseSql(Literal.create(null, StringType)), null)
+ }
+ }
+
+ test("parse_sql does not throw on syntax error") {
+ withSQLConf(SQLConf.PARSE_SQL_ENABLED.key -> "true") {
+ val j = evalJson("NOT A STATEMENT !!!")
+ assert(j \ "parse_success" === JBool(false))
+ assert(j \ "error" \ "errorClass" === JString("PARSE_SYNTAX_ERROR"))
+ }
+ }
+
+ test("parse_sql works with CodegenFallback path") {
+ withSQLConf(SQLConf.PARSE_SQL_ENABLED.key -> "true") {
+ val expr = ParseSql(Literal("INSERT INTO t SELECT 1"))
+ assert(expr.isInstanceOf[CodegenFallback])
+ val j = evalJson("INSERT INTO t SELECT 1")
+ assert(j \ "statement_identifier" === JString("INSERT"))
+ }
+ }
+}
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/catalyst/parser/ParseSqlResultSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/catalyst/parser/ParseSqlResultSuite.scala
new file mode 100644
index 000000000000..2dd031fb2c20
--- /dev/null
+++
b/sql/core/src/test/scala/org/apache/spark/sql/catalyst/parser/ParseSqlResultSuite.scala
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.parser
+
+import org.json4s._
+import org.json4s.jackson.JsonMethods.parse
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.internal.SQLConf
+
+/**
+ * Pin Table 39 codes and contracts that goldens do not cover.
+ * Behavioral coverage lives in sql-tests/inputs/parse-sql.sql.
+ */
+class ParseSqlResultSuite extends SparkFunSuite {
+
+ private def obj(sql: String): JObject =
+ parse(ParseSqlResult.fromSql(sql)).asInstanceOf[JObject]
+
+ private def tableRefs(sql: String, field: String): Set[Seq[String]] =
+ obj(sql) \ field match {
+ case JNothing => Set.empty
+ case JArray(arr) => arr.map {
+ case JArray(parts) => parts.map(_.asInstanceOf[JString].s)
+ case other => fail(s"unexpected $field entry: $other")
+ }.toSet
+ case other => fail(s"unexpected $field: $other")
+ }
+
+ private def sourceTableRefs(sql: String): Set[Seq[String]] =
+ tableRefs(sql, "source_table_references")
+
+ private def targetTableRefs(sql: String): Set[Seq[String]] =
+ tableRefs(sql, "target_table_references")
+
+ test("Table 39 standard and Spark code pairs are pinned") {
+ assert(SqlStatementCodes.Select.statementCode === 21)
+ assert(SqlStatementCodes.Insert.statementCode === 50)
+ assert(SqlStatementCodes.DeleteWhere.statementCode === 19)
+ assert(SqlStatementCodes.UpdateWhere.statementCode === 82)
+ assert(SqlStatementCodes.Merge.statementCode === 128)
+ assert(SqlStatementCodes.CreateTable.statementCode === 77)
+ assert(SqlStatementCodes.CreateView.statementCode === 84)
+ assert(SqlStatementCodes.DropTable.statementCode === 32)
+ assert(SqlStatementCodes.AlterTable.statementCode === 4)
+ assert(SqlStatementCodes.TruncateTable.statementCode === 139)
+ assert(SqlStatementCodes.Unrecognized.statementCode === 0)
+ assert(SqlStatementCodes.CacheTable.statementCode < 0)
+ assert(SqlStatementCodes.BeginEnd.statementCode === -22)
+ assert(SqlStatementCodes.Explain.statementCode === -23)
+ assert(SqlStatementCodes.Set.statementCode === -24)
+ assert(SqlStatementCodes.CreateMetricViewStmt.statementCode === -37)
+ }
+
+ test("TABLE and VALUES classify as SELECT") {
+ val table = obj("TABLE t")
+ assert(table \ "statement_identifier" === JString("SELECT"))
+ assert(table \ "statement_code" === JInt(21))
+ assert(sourceTableRefs("TABLE t") === Set(Seq("t")))
+
+ // Eager inlining must not flip VALUES between SELECT and Unrecognized.
+ Seq(true, false).foreach { eager =>
+ SQLConf.withExistingConf(new SQLConf) {
+
SQLConf.get.setConf(SQLConf.EAGER_EVAL_OF_UNRESOLVED_INLINE_TABLE_ENABLED,
eager)
+ val values = obj("VALUES (1), (2)")
+ assert(values \ "statement_identifier" === JString("SELECT"),
+ s"eager=$eager")
+ assert(values \ "statement_code" === JInt(21), s"eager=$eager")
+ }
+ }
+ }
+
+ test("CREATE FUNCTION and DECLARE VARIABLE are not table references") {
+ assert(sourceTableRefs("CREATE FUNCTION f AS 'x' USING JAR
'y.jar'").isEmpty)
+ assert(targetTableRefs("CREATE FUNCTION f AS 'x' USING JAR
'y.jar'").isEmpty)
+ assert(sourceTableRefs("DECLARE VARIABLE x INT").isEmpty)
+ assert(targetTableRefs("DECLARE VARIABLE x INT").isEmpty)
+ // Contrast: CREATE VIEW still reports the view target and query source.
+ assert(targetTableRefs("CREATE VIEW v AS SELECT 1 AS a") === Set(Seq("v")))
+ assert(sourceTableRefs("CREATE VIEW v AS SELECT 1 AS a").isEmpty)
+ }
+
+ test("DML and DDL split target and source table references") {
+ assert(targetTableRefs("INSERT INTO t SELECT 1") === Set(Seq("t")))
+ assert(sourceTableRefs("INSERT INTO t SELECT 1").isEmpty)
+
+ assert(targetTableRefs("DELETE FROM t WHERE a = 1") === Set(Seq("t")))
+ assert(sourceTableRefs("DELETE FROM t WHERE a = 1").isEmpty)
+
+ assert(targetTableRefs("UPDATE t SET a = 1 WHERE b = 2") === Set(Seq("t")))
+ assert(sourceTableRefs("UPDATE t SET a = 1 WHERE b = 2").isEmpty)
+
+ assert(targetTableRefs(
+ "MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN DELETE") ===
Set(Seq("t")))
+ assert(sourceTableRefs(
+ "MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN DELETE") ===
Set(Seq("s")))
+
+ assert(targetTableRefs("CREATE TABLE x AS SELECT a FROM src") ===
Set(Seq("x")))
+ assert(sourceTableRefs("CREATE TABLE x AS SELECT a FROM src") ===
Set(Seq("src")))
+
+ assert(targetTableRefs("DROP TABLE t") === Set(Seq("t")))
+ assert(sourceTableRefs("DROP TABLE t").isEmpty)
+ }
+
+ test("CTE aliases only shadow references within their own scope") {
+ // The inner CTE named real_t must not hide the outer real table real_t.
+ assert(sourceTableRefs(
+ "SELECT * FROM real_t WHERE EXISTS (" +
+ "WITH real_t AS (SELECT * FROM inner_base) SELECT * FROM real_t)") ===
+ Set(Seq("real_t"), Seq("inner_base")))
+ // A definition sees only preceding aliases, so b here is the real table.
+ assert(sourceTableRefs(
+ "WITH a AS (SELECT * FROM b), b AS (SELECT 1 AS x) SELECT * FROM a") ===
+ Set(Seq("b")))
+ }
+
+ test("positional markers inside BEGIN END are counted once") {
+ val j = obj("BEGIN SELECT * FROM t WHERE a = ?; END")
+ assert(j \ "parse_success" === JBool(true))
+ assert(j \ "statement_identifier" === JString("BEGIN END"))
+ assert(j \ "parameter_markers" \ "unnamed_count" === JInt(1))
+ assert(j \ "parameter_markers" \ "named" === JNothing)
+ assert(sourceTableRefs("BEGIN SELECT * FROM t WHERE a = ?; END") ===
Set(Seq("t")))
+ }
+
+ test("syntax error returns STANDARD error JSON without throwing") {
+ val j = obj("SELEC FROM t")
+ assert(j \ "parse_success" === JBool(false))
+ assert(j \ "error" \ "errorClass" === JString("PARSE_SYNTAX_ERROR"))
+ }
+}
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowFunctionsSuiteBase.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowFunctionsSuiteBase.scala
index 19bd83050083..23d8e3d1f0b3 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowFunctionsSuiteBase.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowFunctionsSuiteBase.scala
@@ -118,6 +118,9 @@ trait ShowFunctionsSuiteBase extends QueryTest with
DDLCommandTestUtils {
assert(!systemFuns.filter("function='case'").isEmpty)
// Built-in functions
assert(!systemFuns.filter("function='substring'").isEmpty)
+ // sql/core-only builtin registered after FunctionRegistry.builtin is
cloned
+ assert(!systemFuns.filter("function='parse_sql'").isEmpty)
+ assert(sql(s"SHOW USER FUNCTIONS IN
$ns").filter("function='parse_sql'").isEmpty)
}
}
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala
index dab117fa6713..7b5402119b40 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala
@@ -292,6 +292,8 @@ class ExpressionInfoSuite extends SharedSparkSession {
val clonedSpark = spark.cloneSession()
// Coalescing partitions can change result order, so disable it.
clonedSpark.conf.set(SQLConf.COALESCE_PARTITIONS_ENABLED.key, false)
+ // parse_sql examples require the experimental feature flag.
+ clonedSpark.conf.set(SQLConf.PARSE_SQL_ENABLED.key, true)
val info = clonedSpark.sessionState.catalog.lookupFunctionInfo(funcId)
val className = info.getClassName
if (!ignoreSet.contains(className)) {
diff --git a/sql/gen-sql-functions-docs.py b/sql/gen-sql-functions-docs.py
index 2ae00f6db822..2585066047aa 100644
--- a/sql/gen-sql-functions-docs.py
+++ b/sql/gen-sql-functions-docs.py
@@ -243,6 +243,8 @@ def generate_functions_examples_html(jvm, jspark,
html_output_dir):
"""
print("Enabling TIME data type")
jspark.sql("SET spark.sql.timeType.enabled = true")
+ print("Enabling parse_sql function")
+ jspark.sql("SET spark.sql.function.parseSql.enabled = true")
print("Running SQL examples to generate formatted output.")
for key, infos in _list_grouped_function_infos(jvm):
examples = _make_pretty_examples(jspark, infos)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
