voonhous commented on code in PR #18961: URL: https://github.com/apache/hudi/pull/18961#discussion_r3820947753
########## hudi-spark-datasource/hudi-spark4.2.x/src/main/scala/org/apache/hudi/variant/Spark42VariantShreddingSchemaInferrer.scala: ########## @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.variant + +import org.apache.hudi.HoodieSchemaConversionUtils +import org.apache.hudi.common.avro.VariantShreddingSchemaInferrer +import org.apache.hudi.common.avro.VariantShreddingSchemaInferrer.VariantSample +import org.apache.hudi.common.schema.HoodieSchema + +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions.GenericInternalRow +import org.apache.spark.sql.execution.datasources.parquet.InferVariantShreddingSchema +import org.apache.spark.sql.types.{ArrayType, DataType, StructField, StructType, VariantType} +import org.apache.spark.unsafe.types.VariantVal + +import java.{util => ju} + +import scala.jdk.CollectionConverters.ListHasAsScala + +/** + * Infers per-file variant shredding schemas by delegating to Spark's + * [[InferVariantShreddingSchema]] (SPARK-53659), so Hudi inherits Spark's merge and + * finalization heuristics verbatim (field-frequency dropping, type widening, width/depth caps). + * + * Loaded reflectively from hudi-common via classpath detection; the Spark 4.2 twin of the + * spark4.1 module's inferrer (each spark4.x profile builds only its own version module, and the + * Spark class does not exist before 4.1). + */ +class Spark42VariantShreddingSchemaInferrer extends VariantShreddingSchemaInferrer { + + private val typedValueField = HoodieSchema.Variant.VARIANT_TYPED_VALUE_FIELD + // Avro identifier shape; the Hudi schema the result splices into is Avro-backed. + private val avroNamePattern = "[A-Za-z_][A-Za-z0-9_]*".r.pattern + + override def inferTypedValueSchemas(columnNames: ju.List[String], + rowSamples: ju.List[Array[VariantSample]]): ju.Map[String, HoodieSchema] = { + val names = columnNames.asScala.toSeq + val inputSchema = StructType(names.map(name => StructField(name, VariantType, nullable = true))) + val rows: Seq[InternalRow] = rowSamples.asScala.map { row => + val values = new Array[Any](row.length) + var i = 0 + while (i < row.length) { + if (row(i) != null) { + values(i) = new VariantVal(row(i).getValue, row(i).getMetadata) + } + i += 1 + } + new GenericInternalRow(values): InternalRow + }.toSeq + + // One call covers all variant columns of the file: Spark's max-width budget is global + // across the schema, and per-column calls would skew it. + val inferred = new InferVariantShreddingSchema(inputSchema).inferSchema(rows) Review Comment: Fixed in 7860db3c9d52: both inferrers resolve the two caps SQLConf-first with a SparkConf fallback (the `resolveSessionLocalTimeZone` pattern) and pin them via `SQLConf.withExistingConf` around the `inferSchema` call. One precision note: inline compaction inside a SQL write still runs under `withSQLConfPropagated`, so it was already honored; offline and async compaction were the dropped cases. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
