This is an automated email from the ASF dual-hosted git repository. voonhous pushed a commit to branch release-1.2.1 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 94ceafb511b5e11d48582cc22b5b2dc0c32c56b2 Author: Lokesh Jain <[email protected]> AuthorDate: Fri Jun 12 05:33:54 2026 +0530 fix(spark): catch HoodieSchemaNotFoundException in 3-arg DefaultSource.createRelation (#18977) fix(spark): catch HoodieSchemaNotFoundException in 3-arg DefaultSource.createRelation The 2-arg createRelation(sqlContext, parameters) overload catches HoodieSchemaNotFoundException and returns an EmptyRelation (HUDI-7147 / #10689), but the 3-arg createRelation(sqlContext, params, schema) overload — which Spark's DataSource.resolveRelation() invokes directly via the SchemaRelationProvider path whenever a user-supplied schema is present (e.g. spark.read.schema(s).format("hudi").load(path), or HMS- catalog resolution that already knows the schema) — has no such catch, so the exception propagates and breaks query analysis. Mirror the catch on the 3-arg overload. Preserve the caller-supplied schema in the EmptyRelation so downstream analysis (e.g. column resolution in WHERE clauses) sees the HMS-known columns even when the on-disk table is schemaless. The 2-arg overload re-enters this method with schema=null, so fall back to an empty StructType in that case to avoid an NPE in the 2-arg overload's relation.schema.isEmpty check (surfaced by TestCOWDataSource.testReadOfAnEmptyTable on spark3.3 / spark3.5). Adds TestCOWDataSource.testReadOfAnEmptyTableWithUserSuppliedSchema, a sibling of testReadOfAnEmptyTable that exercises the 3-arg path. Closes #18668 Co-Authored-By: Claude Opus 4.7 <[email protected]> (cherry picked from commit 182428f803d0432de518ba4de49a71d1a423484a) --- .../main/scala/org/apache/hudi/DefaultSource.scala | 16 +++++++++- .../apache/hudi/functional/TestCOWDataSource.scala | 34 ++++++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DefaultSource.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DefaultSource.scala index bf8dee324f4f..2682de70f1fa 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DefaultSource.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DefaultSource.scala @@ -134,7 +134,21 @@ class DefaultSource extends RelationProvider parameters } - val relation = DefaultSource.createRelation(sqlContext, metaClient, schema, options.toMap) + // Spark's DataSource.resolveRelation() invokes this 3-arg overload directly via the + // SchemaRelationProvider path when a user-supplied schema is present (e.g. + // spark.read.schema(...).load(path)). The 2-arg overload catches + // HoodieSchemaNotFoundException and returns an EmptyRelation, but that catch is bypassed + // on this path, so we mirror the same handling here. Preserve the caller-supplied schema + // so subsequent query analysis (e.g. column resolution in WHERE clauses) sees the + // HMS-known columns even though the on-disk table is schemaless. The 2-arg overload also + // re-enters this method with schema=null, so we must fall back to an empty StructType + // when schema is null to avoid an NPE in the 2-arg overload's relation.schema.isEmpty check. + val relation = try { + DefaultSource.createRelation(sqlContext, metaClient, schema, options.toMap) + } catch { + case _: HoodieSchemaNotFoundException => + new EmptyRelation(sqlContext, Option(schema).getOrElse(new StructType())) + } log.info(s"Created relation ${relation.getClass.getSimpleName} with ${options.size} resolved options") relation } diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala index 106c1f09a7ae..599a105e6d00 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala @@ -2215,6 +2215,40 @@ class TestCOWDataSource extends HoodieSparkClientTestBase with ScalaAssertionSup assertEquals(count, 0) } + @Test + def testReadOfAnEmptyTableWithUserSuppliedSchema(): Unit = { + val (writeOpts, _) = getWriterReaderOpts(HoodieRecordType.AVRO) + + // Insert + then delete the only completed commit so the table has no resolvable schema. + val records = recordsToStrings(dataGen.generateInserts("000", 100)).asScala.toList + val inputDF = spark.read.json(spark.sparkContext.parallelize(records, 2)) + inputDF.write.format("hudi") + .options(writeOpts) + .option(DataSourceWriteOptions.OPERATION.key, DataSourceWriteOptions.INSERT_OPERATION_OPT_VAL) + .mode(SaveMode.Overwrite) + .save(basePath) + + val fileStatuses = storage.listDirectEntries( + new StoragePath(basePath + StoragePath.SEPARATOR + HoodieTableMetaClient.METAFOLDER_NAME + + StoragePath.SEPARATOR + HoodieTableMetaClient.TIMELINEFOLDER_NAME), + new StoragePathFilter { + override def accept(path: StoragePath): Boolean = { + path.getName.endsWith(HoodieTimeline.COMMIT_ACTION) + } + }) + storage.deleteFile(fileStatuses.get(0).getPath) + + // spark.read.schema(...) triggers Spark's SchemaRelationProvider path which calls the + // 3-arg DefaultSource.createRelation overload directly. Without the catch on that + // overload, this would fail with HoodieSchemaNotFoundException. + val userSchema = inputDF.schema + val df = spark.read.schema(userSchema).format("hudi").load(basePath) + assertEquals(0, df.count()) + // The caller-supplied schema must be preserved on the EmptyRelation so subsequent query + // analysis (e.g. column resolution) sees the user-known columns. + assertEquals(userSchema, df.schema) + } + /** * Test incremental queries and time travel queries with event time ordering. *
