This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 07054c00ffe [HUDI-7078] Re-enable TestNestedSchemaPruningOptimization
(#10245)
07054c00ffe is described below
commit 07054c00ffee7ac7dde412dc8aafa83129d7f2d9
Author: Lin Liu <[email protected]>
AuthorDate: Wed Dec 6 14:58:47 2023 -0800
[HUDI-7078] Re-enable TestNestedSchemaPruningOptimization (#10245)
---
.../spark/sql/hudi/TestNestedSchemaPruningOptimization.scala | 11 +++++------
1 file changed, 5 insertions(+), 6 deletions(-)
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestNestedSchemaPruningOptimization.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestNestedSchemaPruningOptimization.scala
index 0c9d213e388..cd4f90da4e7 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestNestedSchemaPruningOptimization.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestNestedSchemaPruningOptimization.scala
@@ -17,6 +17,8 @@
package org.apache.spark.sql.hudi
+import org.apache.hudi.common.config.HoodieCommonConfig
+import org.apache.hudi.config.HoodieWriteConfig
import org.apache.hudi.{HoodieSparkUtils, SparkAdapterSupport}
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.execution.{FileSourceScanExec, ProjectExec,
RowDataSourceScanExec, SparkPlan}
@@ -103,8 +105,6 @@ class TestNestedSchemaPruningOptimization extends
HoodieSparkSqlTestBase with Sp
}
test("Test NestedSchemaPruning optimization unsuccessful") {
- // TODO(HUDI-7078): to revisit with new file format and file group reader
- /*
withTempDir { tmp =>
// NOTE: This tests are only relevant for Spark >= 3.1
// TODO extract tests into a separate spark-version-specific module
@@ -133,11 +133,10 @@ class TestNestedSchemaPruningOptimization extends
HoodieSparkSqlTestBase with Sp
}
val expectedSchema = StructType(Seq(
- StructField("id", IntegerType, nullable = false),
+ StructField("id", IntegerType, nullable = true),
StructField("item",
StructType(Seq(
- StructField("name", StringType, nullable = false),
- StructField("price", IntegerType, nullable = false))),
nullable = false)
+ StructField("name", StringType, nullable = false))), nullable
= true)
))
val expectedReadSchemaClause = "ReadSchema:
struct<id:int,item:struct<name:string,price:int>>"
@@ -174,7 +173,7 @@ class TestNestedSchemaPruningOptimization extends
HoodieSparkSqlTestBase with Sp
selectDF.count
}
}
- }*/
+ }
}
private def createTableWithNestedStructSchema(tableType: String,
