zhangyue19921010 commented on code in PR #8714:
URL: https://github.com/apache/hudi/pull/8714#discussion_r1194548389
##########
hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/HoodieSpark3CatalystExpressionUtils.scala:
##########
@@ -17,16 +17,9 @@
package org.apache.spark.sql
-import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeSet,
Expression, Predicate, PredicateHelper}
-import org.apache.spark.sql.execution.datasources.DataSourceStrategy
-
-trait HoodieSpark3CatalystExpressionUtils extends HoodieCatalystExpressionUtils
- with PredicateHelper {
-
- override def normalizeExprs(exprs: Seq[Expression], attributes:
Seq[Attribute]): Seq[Expression] =
Review Comment:
`DataSourceStrategy.normalizeExprs(exprs, attributes)` are not included in
spark3.0. So that we need to remove this specific implement in spark3-common
space.
##########
hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/HoodieSpark3CatalystExpressionUtils.scala:
##########
@@ -17,16 +17,9 @@
package org.apache.spark.sql
-import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeSet,
Expression, Predicate, PredicateHelper}
-import org.apache.spark.sql.execution.datasources.DataSourceStrategy
-
-trait HoodieSpark3CatalystExpressionUtils extends HoodieCatalystExpressionUtils
- with PredicateHelper {
-
- override def normalizeExprs(exprs: Seq[Expression], attributes:
Seq[Attribute]): Seq[Expression] =
- DataSourceStrategy.normalizeExprs(exprs, attributes)
-
- override def extractPredicatesWithinOutputSet(condition: Expression,
Review Comment:
`extractPredicatesWithinOutputSet` are not included in spark3.0. So that we
need to remove this specific implement in spark3-common space.
##########
hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/adapter/BaseSpark3Adapter.scala:
##########
@@ -108,23 +87,5 @@ abstract class BaseSpark3Adapter extends SparkAdapter with
Logging {
DefaultSource.createRelation(sqlContext, metaClient, dataSchema,
globPaths, parameters.asScala.toMap)
}
- /**
- * Converts instance of [[StorageLevel]] to a corresponding string
- */
- override def convertStorageLevelToString(level: StorageLevel): String =
level match {
- case NONE => "NONE"
- case DISK_ONLY => "DISK_ONLY"
- case DISK_ONLY_2 => "DISK_ONLY_2"
- case DISK_ONLY_3 => "DISK_ONLY_3"
Review Comment:
`DISK_ONLY_3` are not included in spark3.0. So that we need to remove this
specific implement in spark3-common space.
##########
hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/adapter/BaseSpark3Adapter.scala:
##########
@@ -63,20 +56,6 @@ abstract class BaseSpark3Adapter extends SparkAdapter with
Logging {
new Spark3RowSerDe(encoder)
}
- override def resolveHoodieTable(plan: LogicalPlan): Option[CatalogTable] = {
- super.resolveHoodieTable(plan).orElse {
- EliminateSubqueryAliases(plan) match {
- // First, we need to weed out unresolved plans
- case plan if !plan.resolved => None
- // NOTE: When resolving Hudi table we allow [[Filter]]s and
[[Project]]s be applied
- // on top of it
- case PhysicalOperation(_, _, DataSourceV2Relation(v2:
V2TableWithV1Fallback, _, _, _, _)) if isHoodieTable(v2.v1Table) =>
Review Comment:
`V2TableWithV1Fallback` are not included in spark3.0. So that we need to
remove this specific implement in spark3-common space.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
