Repository: spark Updated Branches: refs/heads/master b208229ba -> 162cf02ef
[SPARK-14910][SQL] Native DDL Command Support for Describe Function in Non-identifier Format #### What changes were proposed in this pull request? The existing `Describe Function` only support the function name in `identifier`. This is different from what Hive behaves. That is why many test cases `udf_abc` in `HiveCompatibilitySuite` are not using our native DDL support. For example, - udf_not.q - udf_bitwise_not.q This PR is to resolve the issues. Now, we can support the command of `Describe Function` whose function names are in the following format: - `qualifiedName` (e.g., `db.func1`) - `STRING` (e.g., `'func1'`) - `comparisonOperator` (e.g,. `<`) - `arithmeticOperator` (e.g., `+`) - `predicateOperator` (e.g., `or`) Note, before this PR, we only have a native command support when the function name is in the format of `qualifiedName`. #### How was this patch tested? Added test cases in `DDLSuite.scala`. Also manually verified all the related test cases in `HiveCompatibilitySuite` passed. Author: gatorsmile <[email protected]> Closes #12679 from gatorsmile/descFunction. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/162cf02e Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/162cf02e Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/162cf02e Branch: refs/heads/master Commit: 162cf02efa025fdb32adc3eaabb8e4232fe90e08 Parents: b208229 Author: gatorsmile <[email protected]> Authored: Tue Apr 26 19:29:34 2016 +0200 Committer: Herman van Hovell <[email protected]> Committed: Tue Apr 26 19:29:34 2016 +0200 ---------------------------------------------------------------------- .../apache/spark/sql/catalyst/parser/SqlBase.g4 | 18 ++++++- .../catalyst/analysis/FunctionRegistry.scala | 1 + .../spark/sql/catalyst/parser/AstBuilder.scala | 12 ++++- .../spark/sql/execution/command/DDLSuite.scala | 51 ++++++++++++++++++++ 4 files changed, 79 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/162cf02e/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 index fa4b8c4..64f68c9 100644 --- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 +++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 @@ -107,7 +107,7 @@ statement | SHOW TBLPROPERTIES table=tableIdentifier ('(' key=tablePropertyKey ')')? #showTblProperties | SHOW FUNCTIONS (LIKE? (qualifiedName | pattern=STRING))? #showFunctions - | (DESC | DESCRIBE) FUNCTION EXTENDED? qualifiedName #describeFunction + | (DESC | DESCRIBE) FUNCTION EXTENDED? describeFuncName #describeFunction | (DESC | DESCRIBE) option=(EXTENDED | FORMATTED)? tableIdentifier partitionSpec? describeColName? #describeTable | (DESC | DESCRIBE) DATABASE EXTENDED? identifier #describeDatabase @@ -220,6 +220,14 @@ partitionVal : identifier (EQ constant)? ; +describeFuncName + : qualifiedName + | STRING + | comparisonOperator + | arithmeticOperator + | predicateOperator + ; + describeColName : identifier ('.' (identifier | STRING))* ; @@ -519,6 +527,14 @@ comparisonOperator : EQ | NEQ | NEQJ | LT | LTE | GT | GTE | NSEQ ; +arithmeticOperator + : PLUS | MINUS | ASTERISK | SLASH | PERCENT | DIV | TILDE | AMPERSAND | PIPE | HAT + ; + +predicateOperator + : OR | AND | IN | NOT + ; + booleanValue : TRUE | FALSE ; http://git-wip-us.apache.org/repos/asf/spark/blob/162cf02e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala index dd35770..1bada2c 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala @@ -363,6 +363,7 @@ object FunctionRegistry { expression[Not]("not"), expression[Or]("or"), + // comparison operators expression[EqualNullSafe]("<=>"), expression[EqualTo]("="), expression[EqualTo]("=="), http://git-wip-us.apache.org/repos/asf/spark/blob/162cf02e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index 3d7b888..7f98c21 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -112,8 +112,16 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with Logging { * Create a plan for a DESCRIBE FUNCTION command. */ override def visitDescribeFunction(ctx: DescribeFunctionContext): LogicalPlan = withOrigin(ctx) { - val functionName = ctx.qualifiedName().identifier().asScala.map(_.getText).mkString(".") - DescribeFunction(functionName, ctx.EXTENDED != null) + import ctx._ + val functionName = + if (describeFuncName.STRING() != null) { + string(describeFuncName.STRING()) + } else if (describeFuncName.qualifiedName() != null) { + describeFuncName.qualifiedName().identifier().asScala.map(_.getText).mkString(".") + } else { + describeFuncName.getText + } + DescribeFunction(functionName, EXTENDED != null) } /** http://git-wip-us.apache.org/repos/asf/spark/blob/162cf02e/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala index 9ffffa0..e601ff1 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala @@ -716,4 +716,55 @@ class DDLSuite extends QueryTest with SharedSQLContext with BeforeAndAfterEach { } } + test("describe function") { + checkAnswer( + sql("DESCRIBE FUNCTION log"), + Row("Class: org.apache.spark.sql.catalyst.expressions.Logarithm") :: + Row("Function: log") :: + Row("Usage: log(b, x) - Returns the logarithm of x with base b.") :: Nil + ) + // predicate operator + checkAnswer( + sql("DESCRIBE FUNCTION or"), + Row("Class: org.apache.spark.sql.catalyst.expressions.Or") :: + Row("Function: or") :: + Row("Usage: a or b - Logical OR.") :: Nil + ) + checkAnswer( + sql("DESCRIBE FUNCTION !"), + Row("Class: org.apache.spark.sql.catalyst.expressions.Not") :: + Row("Function: !") :: + Row("Usage: ! a - Logical not") :: Nil + ) + // arithmetic operators + checkAnswer( + sql("DESCRIBE FUNCTION +"), + Row("Class: org.apache.spark.sql.catalyst.expressions.Add") :: + Row("Function: +") :: + Row("Usage: a + b - Returns a+b.") :: Nil + ) + // comparison operators + checkAnswer( + sql("DESCRIBE FUNCTION <"), + Row("Class: org.apache.spark.sql.catalyst.expressions.LessThan") :: + Row("Function: <") :: + Row("Usage: a < b - Returns TRUE if a is less than b.") :: Nil + ) + // STRING + checkAnswer( + sql("DESCRIBE FUNCTION 'concat'"), + Row("Class: org.apache.spark.sql.catalyst.expressions.Concat") :: + Row("Function: concat") :: + Row("Usage: concat(str1, str2, ..., strN) " + + "- Returns the concatenation of str1, str2, ..., strN") :: Nil + ) + // extended mode + checkAnswer( + sql("DESCRIBE FUNCTION EXTENDED ^"), + Row("Class: org.apache.spark.sql.catalyst.expressions.BitwiseXor") :: + Row("Extended Usage:\n> SELECT 3 ^ 5; 2") :: + Row("Function: ^") :: + Row("Usage: a ^ b - Bitwise exclusive OR.") :: Nil + ) + } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
