alexeykudinkin commented on code in PR #5737:
URL: https://github.com/apache/hudi/pull/5737#discussion_r891768307
##########
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieAnalysis.scala:
##########
@@ -39,45 +39,69 @@ import org.apache.spark.sql.{AnalysisException,
SparkSession}
import java.util
import scala.collection.JavaConverters._
+import scala.collection.mutable.ListBuffer
object HoodieAnalysis {
- def customResolutionRules(): Seq[SparkSession => Rule[LogicalPlan]] =
- Seq(
+ type RuleBuilder = SparkSession => Rule[LogicalPlan]
+
+ def customResolutionRules: Seq[RuleBuilder] = {
+ val rules: ListBuffer[RuleBuilder] = ListBuffer(
+ // Default rules
session => HoodieResolveReferences(session),
session => HoodieAnalysis(session)
- ) ++ extraResolutionRules()
-
- def customPostHocResolutionRules(): Seq[SparkSession => Rule[LogicalPlan]] =
- Seq(
- session => HoodiePostAnalysisRule(session)
- ) ++ extraPostHocResolutionRules()
+ )
- def extraResolutionRules(): Seq[SparkSession => Rule[LogicalPlan]] = {
if (HoodieSparkUtils.gteqSpark3_2) {
+ val dataSourceV2ToV1FallbackClass =
"org.apache.spark.sql.hudi.analysis.HoodieDataSourceV2ToV1Fallback"
+ val dataSourceV2ToV1Fallback: RuleBuilder =
+ session => ReflectionUtils.loadClass(dataSourceV2ToV1FallbackClass,
session).asInstanceOf[Rule[LogicalPlan]]
+
val spark3AnalysisClass =
"org.apache.spark.sql.hudi.analysis.HoodieSpark3Analysis"
- val spark3Analysis: SparkSession => Rule[LogicalPlan] =
+ val spark3Analysis: RuleBuilder =
session => ReflectionUtils.loadClass(spark3AnalysisClass,
session).asInstanceOf[Rule[LogicalPlan]]
- val spark3ResolveReferences =
"org.apache.spark.sql.hudi.analysis.HoodieSpark3ResolveReferences"
- val spark3References: SparkSession => Rule[LogicalPlan] =
- session => ReflectionUtils.loadClass(spark3ResolveReferences,
session).asInstanceOf[Rule[LogicalPlan]]
+ val spark3ResolveReferencesClass =
"org.apache.spark.sql.hudi.analysis.HoodieSpark3ResolveReferences"
+ val spark3ResolveReferences: RuleBuilder =
+ session => ReflectionUtils.loadClass(spark3ResolveReferencesClass,
session).asInstanceOf[Rule[LogicalPlan]]
- Seq(spark3Analysis, spark3References)
- } else {
- Seq.empty
+ val spark32ResolveAlterTableCommandsClass =
"org.apache.spark.sql.hudi.ResolveHudiAlterTableCommandSpark32"
+ val spark32ResolveAlterTableCommands: RuleBuilder =
+ session =>
ReflectionUtils.loadClass(spark32ResolveAlterTableCommandsClass,
session).asInstanceOf[Rule[LogicalPlan]]
+
+ // NOTE: PLEASE READ CAREFULLY
+ //
+ // It's critical for this rules to follow in this order, so that
DataSource V2 to V1 fallback
+ // is performed prior to other rules being evaluated
+ rules ++= Seq(dataSourceV2ToV1Fallback, spark3Analysis,
spark3ResolveReferences, spark32ResolveAlterTableCommands)
Review Comment:
Yeah, we need to properly align naming in quite a few places
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
