yeshengm commented on a change in pull request #24809: [SPARK-21136][SQL]
Disallow FROM-only statements and show better warnings for Hive-style
single-from statements
URL: https://github.com/apache/spark/pull/24809#discussion_r291374618
##########
File path:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
##########
@@ -403,140 +420,177 @@ class AstBuilder(conf: SQLConf) extends
SqlBaseBaseVisitor[AnyRef] with Logging
throw new ParseException("DISTRIBUTE BY is not supported", ctx)
}
- /**
- * Create a logical plan using a query specification.
- */
- override def visitQuerySpecification(
- ctx: QuerySpecificationContext): LogicalPlan = withOrigin(ctx) {
+ override def visitTransformQuerySpecification(
+ ctx: TransformQuerySpecificationContext): LogicalPlan = withOrigin(ctx) {
val from = OneRowRelation().optional(ctx.fromClause) {
visitFromClause(ctx.fromClause)
}
- withQuerySpecification(ctx, from)
+ withTransformQuerySpecification(ctx, ctx.transformClause, ctx.whereClause,
from)
}
- /**
- * Add a query specification to a logical plan. The query specification is
the core of the logical
- * plan, this is where sourcing (FROM clause), transforming (SELECT
TRANSFORM/MAP/REDUCE),
- * projection (SELECT), aggregation (GROUP BY ... HAVING ...) and filtering
(WHERE) takes place.
- *
- * Note that query hints are ignored (both by the parser and the builder).
- */
- private def withQuerySpecification(
- ctx: QuerySpecificationContext,
- relation: LogicalPlan): LogicalPlan = withOrigin(ctx) {
- import ctx._
-
- // WHERE
- def filter(ctx: BooleanExpressionContext, plan: LogicalPlan): LogicalPlan
= {
- Filter(expression(ctx), plan)
- }
-
- def withHaving(ctx: BooleanExpressionContext, plan: LogicalPlan):
LogicalPlan = {
- // Note that we add a cast to non-predicate expressions. If the
expression itself is
- // already boolean, the optimizer will get rid of the unnecessary cast.
- val predicate = expression(ctx) match {
- case p: Predicate => p
- case e => Cast(e, BooleanType)
- }
- Filter(predicate, plan)
+ override def visitRegularQuerySpecification(
+ ctx: RegularQuerySpecificationContext): LogicalPlan = withOrigin(ctx) {
+ val from = OneRowRelation().optional(ctx.fromClause) {
+ visitFromClause(ctx.fromClause)
}
+ withSelectQuerySpecification(
+ ctx,
+ ctx.selectClause,
+ ctx.lateralView,
+ ctx.whereClause,
+ ctx.aggregationClause,
+ ctx.havingClause,
+ ctx.windowClause,
+ from
+ )
+ }
-
- // Expressions.
- val expressions = Option(namedExpressionSeq).toSeq
+ override def visitNamedExpressionSeq(
+ ctx: NamedExpressionSeqContext): Seq[Expression] = {
+ Option(ctx).toSeq
.flatMap(_.namedExpression.asScala)
.map(typedVisit[Expression])
+ }
- // Create either a transform or a regular query.
- val specType = Option(kind).map(_.getType).getOrElse(SqlBaseParser.SELECT)
- specType match {
- case SqlBaseParser.MAP | SqlBaseParser.REDUCE | SqlBaseParser.TRANSFORM
=>
- // Transform
-
- // Add where.
- val withFilter = relation.optionalMap(where)(filter)
-
- // Create the attributes.
- val (attributes, schemaLess) = if (colTypeList != null) {
- // Typed return columns.
- (createSchema(colTypeList).toAttributes, false)
- } else if (identifierSeq != null) {
- // Untyped return columns.
- val attrs = visitIdentifierSeq(identifierSeq).map { name =>
- AttributeReference(name, StringType, nullable = true)()
- }
- (attrs, false)
- } else {
- (Seq(AttributeReference("key", StringType)(),
- AttributeReference("value", StringType)()), true)
- }
+ /**
+ * Create a logical plan using a having clause.
+ */
+ private def withHavingClause(
+ ctx: HavingClauseContext, plan: LogicalPlan): LogicalPlan = {
+ // Note that we add a cast to non-predicate expressions. If the expression
itself is
+ // already boolean, the optimizer will get rid of the unnecessary cast.
+ val predicate = expression(ctx.booleanExpression) match {
+ case p: Predicate => p
+ case e => Cast(e, BooleanType)
+ }
+ Filter(predicate, plan)
+ }
- // Create the transform.
- ScriptTransformation(
- expressions,
- string(script),
- attributes,
- withFilter,
- withScriptIOSchema(
- ctx, inRowFormat, recordWriter, outRowFormat, recordReader,
schemaLess))
+ /**
+ * Create a logical plan using a where clause.
+ */
+ private def withWhereClause(ctx: WhereClauseContext, plan: LogicalPlan):
LogicalPlan = {
+ Filter(expression(ctx.booleanExpression), plan)
+ }
- case SqlBaseParser.SELECT =>
- // Regular select
+ /**
Review comment:
The core method to generate the query plan for a query specification. These
methods have multiple arguments because they need to be reused in both regular
queries and hive-style queries. I can't think of an elegant way of handling
this.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
