This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 93f1f711e065 refactor(schema): schema util contracts, dead API and
folds (#19809)
93f1f711e065 is described below
commit 93f1f711e065e704d06b26f837865933a0e43208
Author: voonhous <[email protected]>
AuthorDate: Thu Sep 3 18:09:28 2026 +0800
refactor(schema): schema util contracts, dead API and folds (#19809)
refactor(schema): schema util contracts, dead API and folds
Part 1 of #16639. The HoodieSchema migration moved schema helpers out
of AvroSchemaUtils and HoodieAvroUtils but never wrote down what
belongs where, so helpers kept being re-implemented instead of found.
Behavior-preserving.
The routing rule, now in the class javadocs: HoodieSchema answers
questions about one schema and holds the factories; HoodieSchemaUtils
holds table-schema transforms and the Hudi record shapes;
HoodieSchemaCompatibility is the only compatibility and projection
entry point; HoodieAvroUtils holds Avro record and value operations;
common.schema.internal is the InternalSchema domain. Look-alike
helpers with different semantics cross-reference each other instead
of being merged.
Code changes to match it:
- HoodieSchemas folds into HoodieSchemaUtils (createDeleteLogSchema).
- AvroSchemaCache deleted: its one caller interned a generated-class
static, a no-op since #18967 removed the last consumer.
- LocalHoodieSchemaCache moves from common.util to common.schema;
getInstance() becomes create(), which is what it always did.
- convertBytesToBigDecimal(byte[], HoodieSchema) moves to
HoodieAvroUtils; the (byte[], int, int) shim and the caller-less
HoodieSchemaUtils.convertValueForSpecificDataTypes and
hasSmallPrecisionDecimalField are deleted.
- HoodieAvroUtils.getRecordColumnValues takes HoodieSchema, since both
callers unwrapped one to call it; it still interns through
HoodieAvroSchemaCache and now reuses the static PROPERTIES and a
fixed array instead of allocating per call.
- 13 caller-less members narrowed; test-only ones get
@VisibleForTesting.
- Stale "equivalent to X" javadocs fixed, cross-references added.
Tests pin delete-log field order and logical types, case-insensitive
projection, nested column values, the HoodieMetadataPayload
reference-equality fast path, the getRecordColumnValues intern, and
LocalHoodieSchemaCache (test restored from #17740).
Pre-existing bugs found on the way are filed, not fixed here:
#19823, #19825, #19826.
Part 2 (#19810) implements asNullable natively, dissolves
AvroSchemaUtils and dedupes the compatibility cluster.
Closes #15908
---
.../hudi/io/cdc/HoodieNativeLogFormatWriter.java | 4 +-
.../TestJavaBulkInsertInternalPartitioner.java | 6 +-
.../apache/hudi/HoodieSchemaConversionUtils.scala | 3 +-
.../apache/spark/sql/HoodieInternalRowUtils.scala | 6 +-
.../apache/hudi/common/avro/AvroSchemaCache.java | 47 ----
.../apache/hudi/common/avro/AvroSchemaUtils.java | 13 +-
.../apache/hudi/common/avro/HoodieAvroUtils.java | 123 +++++++---
.../hudi/common/avro/HoodieAvroWrapperUtils.java | 14 +-
.../processors/DecimalLogicalTypeProcessor.java | 4 +-
.../apache/hudi/common/engine/RecordContext.java | 4 +-
.../apache/hudi/common/model/HoodieAvroRecord.java | 2 +-
.../schema/HoodieSchemaCompatibilityChecker.java | 2 +-
.../hudi/common/schema/HoodieSchemaRepair.java | 19 ++
.../common/schema/HoodieSchemaTypePromotion.java | 38 +--
.../hudi/common/schema/HoodieSchemaUtils.java | 263 +++++++++++----------
.../apache/hudi/common/schema/HoodieSchemas.java | 52 ----
.../{util => schema}/LocalHoodieSchemaCache.java | 11 +-
.../internal/convert/InternalSchemaConverter.java | 2 +-
.../log/block/HoodieNativeLogDeleteBlock.java | 4 +-
.../common/table/read/lsm/LsmFileIterators.java | 4 +-
.../hudi/metadata/HoodieMetadataPayload.java | 8 +-
.../hudi/metadata/HoodieTableMetadataUtil.java | 2 +-
.../hudi/common/avro/TestHoodieAvroUtils.java | 164 +++++++++++++
.../hudi/common/schema/TestHoodieSchemaUtils.java | 211 +++++++----------
.../common/schema/TestLocalHoodieSchemaCache.java | 86 +++++++
.../read/lsm/TestLsmFileGroupRecordIterator.java | 15 --
.../table/read/lsm/TestLsmFileIterators.java | 4 +-
.../testutils/reader/HoodieFileSliceTestUtils.java | 4 +-
.../hudi/metadata/TestHoodieMetadataPayload.java | 26 ++
.../org/apache/hudi/HoodieCreateRecordUtils.scala | 2 +-
.../org/apache/hudi/TestDataSourceDefaults.scala | 15 +-
.../sql/hudi/common/HoodieSparkSqlTestBase.scala | 1 -
.../utilities/sources/TestJsonKafkaSource.java | 3 +-
33 files changed, 709 insertions(+), 453 deletions(-)
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/cdc/HoodieNativeLogFormatWriter.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/cdc/HoodieNativeLogFormatWriter.java
index c23204b9c8fc..c45dc2fa439b 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/cdc/HoodieNativeLogFormatWriter.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/cdc/HoodieNativeLogFormatWriter.java
@@ -25,7 +25,7 @@ import org.apache.hudi.common.model.HoodieFileFormat;
import org.apache.hudi.common.model.HoodieLogFile;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.schema.HoodieSchema;
-import org.apache.hudi.common.schema.HoodieSchemas;
+import org.apache.hudi.common.schema.HoodieSchemaUtils;
import org.apache.hudi.common.table.HoodieTableVersion;
import org.apache.hudi.common.table.log.AppendResult;
import org.apache.hudi.common.table.log.HoodieLogFormat;
@@ -258,7 +258,7 @@ public class HoodieNativeLogFormatWriter extends
HoodieLogFormat.Writer {
ensureAppendVersion();
if (deleteFileWriter == null) {
deleteLogFile = createNativeLogFile(currentAppendVersion,
DELETE_LOG_EXTENSION);
- deleteLogSchema = HoodieSchemas.createDeleteLogSchema(tableSchema,
orderingFieldNames);
+ deleteLogSchema = HoodieSchemaUtils.createDeleteLogSchema(tableSchema,
orderingFieldNames);
// The delete records are built through
recordContext#constructEngineRecord (see #appendDeleteRecord), so the
// writer must match the record context's engine type rather than the
merger's record type: on Spark executors
// the reader context for write degrades to Avro (no engine context
available), and using the merger record
diff --git
a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/execution/bulkinsert/TestJavaBulkInsertInternalPartitioner.java
b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/execution/bulkinsert/TestJavaBulkInsertInternalPartitioner.java
index b7d851aee79b..b8fcd04aae6f 100644
---
a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/execution/bulkinsert/TestJavaBulkInsertInternalPartitioner.java
+++
b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/execution/bulkinsert/TestJavaBulkInsertInternalPartitioner.java
@@ -21,6 +21,7 @@ package org.apache.hudi.execution.bulkinsert;
import org.apache.hudi.common.avro.HoodieAvroUtils;
import org.apache.hudi.common.model.HoodieRecord;
+import org.apache.hudi.common.schema.HoodieSchema;
import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.FlatLists;
@@ -29,7 +30,6 @@ import org.apache.hudi.keygen.constant.KeyGeneratorOptions;
import org.apache.hudi.table.BulkInsertPartitioner;
import org.apache.hudi.testutils.HoodieJavaClientTestHarness;
-import org.apache.avro.Schema;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
@@ -63,7 +63,7 @@ public class TestJavaBulkInsertInternalPartitioner extends
HoodieJavaClientTestH
public void testCustomColumnSortPartitioner(String sortColumnString) throws
Exception {
String[] sortColumns = sortColumnString.split(",");
Comparator<HoodieRecord> columnComparator =
- getCustomColumnComparator(HoodieTestDataGenerator.AVRO_SCHEMA,
sortColumns);
+ getCustomColumnComparator(HoodieTestDataGenerator.HOODIE_SCHEMA,
sortColumns);
List<HoodieRecord> records = generateTestRecordsForBulkInsert(1000);
HoodieWriteConfig cfg =
HoodieWriteConfig.newBuilder().withPath("basePath").build();
@@ -74,7 +74,7 @@ public class TestJavaBulkInsertInternalPartitioner extends
HoodieJavaClientTestH
records, true, generatePartitionNumRecords(records),
Option.of(columnComparator));
}
- private Comparator<HoodieRecord> getCustomColumnComparator(Schema schema,
String[] sortColumns) {
+ private Comparator<HoodieRecord> getCustomColumnComparator(HoodieSchema
schema, String[] sortColumns) {
return Comparator.comparing(record ->
FlatLists.ofComparableArray(
HoodieAvroUtils.getRecordColumnValues(record, sortColumns, schema,
false)));
diff --git
a/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/HoodieSchemaConversionUtils.scala
b/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/HoodieSchemaConversionUtils.scala
index 93baf250bf3a..466f166a5d84 100644
---
a/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/HoodieSchemaConversionUtils.scala
+++
b/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/HoodieSchemaConversionUtils.scala
@@ -345,8 +345,7 @@ object HoodieSchemaConversionUtils {
/**
* Gets the fully-qualified Avro record name and namespace for a Hudi table
- * This delegates to [[HoodieSchemaUtils.getRecordQualifiedName]] which in
turn
- * delegates to [[AvroSchemaUtils.getAvroRecordQualifiedName]].
+ * This delegates to [[HoodieSchemaUtils.getRecordQualifiedName]].
*
* The qualified name follows the pattern:
hoodie.{tableName}.{tableName}_record
* where tableName is sanitized for Avro compatibility.
diff --git
a/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/HoodieInternalRowUtils.scala
b/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/HoodieInternalRowUtils.scala
index 23054564f53e..2cf48747573d 100644
---
a/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/HoodieInternalRowUtils.scala
+++
b/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/HoodieInternalRowUtils.scala
@@ -122,9 +122,9 @@ object HoodieInternalRowUtils {
}
/**
- * Get or create [[StructType]] for provided [[Schema]]
- * @param schema [[Schema]] to convert to [[StructType]], NOTE: It is best
that the schema passed in is cached through
[[org.apache.hudi.common.avro.AvroSchemaCache]], so that we can reduce the
overhead of schema lookup in the map
- * @return [[StructType]] for provided [[Schema]]
+ * Get or create [[StructType]] for provided [[HoodieSchema]]
+ * @param schema [[HoodieSchema]] to convert to [[StructType]], NOTE: It is
best that the schema passed in is cached through
[[org.apache.hudi.common.schema.HoodieSchemaCache]], so that we can reduce the
overhead of schema lookup in the map
+ * @return [[StructType]] for provided [[HoodieSchema]]
*/
def getCachedSchema(schema: HoodieSchema): StructType = {
val structType = schemaMap.get(schema)
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/avro/AvroSchemaCache.java
b/hudi-common/src/main/java/org/apache/hudi/common/avro/AvroSchemaCache.java
deleted file mode 100644
index 0e4573172476..000000000000
--- a/hudi-common/src/main/java/org/apache/hudi/common/avro/AvroSchemaCache.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hudi.common.avro;
-
-import com.github.benmanes.caffeine.cache.Caffeine;
-import com.github.benmanes.caffeine.cache.LoadingCache;
-import org.apache.avro.Schema;
-
-/**
- * An avro schema cache implementation for reusing avro schema instantces in
JVM/process scope.
- * This is a global cache which works for a JVM lifecycle.
- * A collection of schema instants are maintained.
- *
- * <p> NOTE: The schema which be used frequently should be cached through this
cache.
- */
-public class AvroSchemaCache {
-
-
- // Ensure that there is only one variable instance of the same schema within
an entire JVM lifetime
- private static final LoadingCache<Schema, Schema> SCHEMA_CACHE =
Caffeine.newBuilder().weakValues().maximumSize(1024).build(k -> k);
-
- /**
- * Get schema variable from global cache. If not found, put it into the
cache and then return it.
- * @param schema schema to get
- * @return if found, return the exist schema variable, otherwise return the
param itself.
- */
- public static Schema intern(Schema schema) {
- return SCHEMA_CACHE.get(schema);
- }
-
-}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/avro/AvroSchemaUtils.java
b/hudi-common/src/main/java/org/apache/hudi/common/avro/AvroSchemaUtils.java
index e4b90b989a30..1b12c3840456 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/avro/AvroSchemaUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/avro/AvroSchemaUtils.java
@@ -39,7 +39,14 @@ import static
org.apache.hudi.common.util.CollectionUtils.reduce;
import static org.apache.hudi.common.util.ValidationUtils.checkState;
/**
- * Utils for Avro Schema.
+ * Avro-typed schema helpers, retained only as the delegate target of the call
sites that have not moved to
+ * HoodieSchema yet: {@link
org.apache.hudi.common.schema.HoodieSchemaUtils#asNullable(HoodieSchema)} and
+ * {@code HoodieSchemaUtils#createNullableSchema}, the field construction
inside {@link HoodieSchema.Blob},
+ * and a handful of internal uses in {@link HoodieAvroUtils}.
+ *
+ * <p>This class is being retired under #16639. Do not add methods here: every
method on this class except
+ * {@link #getNonNullTypeFromUnion(Schema)} already has a HoodieSchema twin,
so use
+ * {@link org.apache.hudi.common.schema.HoodieSchema} or {@link
org.apache.hudi.common.schema.HoodieSchemaUtils} instead.</p>
*/
@NoArgsConstructor(access = AccessLevel.PRIVATE)
@Slf4j
@@ -87,6 +94,10 @@ public class AvroSchemaUtils {
/**
* Resolves typical Avro's nullable schema definition: {@code
Union(Schema.Type.NULL, <NonNullType>)},
* decomposing union and returning the target non-null type
+ * <p>
+ * This is the strict variant: it throws unless the union has exactly one
null branch and one non-null
+ * branch. See the union-unwrapping note on {@link HoodieAvroUtils} for the
lenient alternatives.
+ * </p>
*/
public static Schema getNonNullTypeFromUnion(Schema schema) {
if (schema.getType() != Schema.Type.UNION) {
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/avro/HoodieAvroUtils.java
b/hudi-common/src/main/java/org/apache/hudi/common/avro/HoodieAvroUtils.java
index de449eab8db1..1a83ba67fa63 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/avro/HoodieAvroUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/avro/HoodieAvroUtils.java
@@ -25,6 +25,7 @@ import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecordPayload;
import org.apache.hudi.common.schema.HoodieAvroSchemaCache;
import org.apache.hudi.common.schema.HoodieSchema;
+import org.apache.hudi.common.schema.HoodieSchemaType;
import org.apache.hudi.common.schema.HoodieSchemaUtils;
import org.apache.hudi.common.util.DateTimeUtils;
import org.apache.hudi.common.util.HoodieRecordUtils;
@@ -32,6 +33,7 @@ import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.OrderingValues;
import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.common.util.ValidationUtils;
+import org.apache.hudi.common.util.VisibleForTesting;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieAvroSchemaException;
import org.apache.hudi.exception.HoodieException;
@@ -112,7 +114,44 @@ import static
org.apache.hudi.common.util.StringUtils.getUTF8Bytes;
import static org.apache.hudi.common.util.ValidationUtils.checkState;
/**
- * Helper class to do common stuff across Avro.
+ * Operations on Avro records and values.
+ *
+ * <p>This class owns the record-level half of the Avro world:</p>
+ * <ul>
+ * <li>serialization: {@code avroToBytes} / {@code bytesToAvro} and the JSON
converters</li>
+ * <li>record rewriting between schemas: {@code rewriteRecord*}, {@code
projectRecordToNewSchemaShallow},
+ * {@code stitchRecords}</li>
+ * <li>value coercion for logical types: {@code convertValueFor*}, {@code
convertBytesToBigDecimal}</li>
+ * <li>record-key, metadata-column and nested-value access: {@code
getNestedFieldVal},
+ * {@code addHoodieKeyToRecord}, {@code getRecordColumnValues}, {@code
createHoodieRecordFromAvro}</li>
+ * </ul>
+ *
+ * <p>Raw {@link Schema} helpers live here only when they serve a record
operation on this class (for example
+ * {@code createNewSchemaField}, {@code unwrapNullable}). This is not the
place for table-schema
+ * manipulation.</p>
+ *
+ * <p>Where else to look:</p>
+ * <ul>
+ * <li>{@link org.apache.hudi.common.schema.HoodieSchema} - questions about
one schema: navigation,
+ * nullability, type predicates, and the {@code create*} / {@code parse}
factories</li>
+ * <li>{@link org.apache.hudi.common.schema.HoodieSchemaUtils} -
HoodieSchema-typed structural transforms
+ * of table schemas</li>
+ * <li>{@link org.apache.hudi.common.schema.HoodieSchemaCompatibility} -
reader/writer compatibility and
+ * projection checks</li>
+ * <li>{@code org.apache.hudi.common.schema.internal} - the field-id
InternalSchema (schema-on-read) domain</li>
+ * </ul>
+ *
+ * <p>Union unwrapping is deliberately not one helper. Pick by contract:</p>
+ * <ul>
+ * <li>{@code unwrapNullable} (this class) - lenient: the first non-null
branch of any union</li>
+ * <li>{@code getActualSchemaFromUnion} (this class, private) - resolves
complex unions against the datum</li>
+ * <li>{@code AvroSchemaUtils#getNonNullTypeFromUnion} - strict: throws
unless the union is exactly one null
+ * branch and one non-null branch</li>
+ * <li>{@link HoodieSchema#getNonNullType()} - strips null branches and
never throws</li>
+ * <li>{@code HoodieSchemaUtils#resolveUnionSchema} - selects a branch by
full name</li>
+ * <li>{@code AvroOrcUtils#getActualSchemaType} - maps an all-null union to
NULL</li>
+ * </ul>
+ * <p>These semantics differ on purpose (see #19212) and must not be merged
into one helper.</p>
*/
@Slf4j
public class HoodieAvroUtils {
@@ -189,7 +228,7 @@ public class HoodieAvroUtils {
return indexedRecordToBytesStream(record);
}
- public static <T extends IndexedRecord> ByteArrayOutputStream
indexedRecordToBytesStream(T record) {
+ private static <T extends IndexedRecord> ByteArrayOutputStream
indexedRecordToBytesStream(T record) {
GenericDatumWriter<T> writer = new
GenericDatumWriter<>(record.getSchema(), ConvertingGenericData.INSTANCE);
try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out,
BINARY_ENCODER.get());
@@ -221,13 +260,12 @@ public class HoodieAvroUtils {
* @param record The GenericRecord to convert
* @param pretty Whether to pretty-print the json output
*/
- public static String avroToJsonString(GenericRecord record, boolean pretty)
throws IOException {
+ private static String avroToJsonString(GenericRecord record, boolean pretty)
throws IOException {
return avroToJsonHelper(record, pretty).toString();
}
/**
* Convert a given avro record to a JSON string. If the record contents are
invalid, return the record.toString().
- * Use this method over {@link HoodieAvroUtils#avroToJsonString} when simply
trying to print the record contents without any guarantees around their
correctness.
*
* @param record The GenericRecord to convert
* @return a JSON string
@@ -359,6 +397,7 @@ public class HoodieAvroUtils {
return AvroSchemaUtils.createNewSchemaFromFieldsWithReference(schema,
filteredFields);
}
+ @VisibleForTesting
public static Schema makeFieldNonNull(Schema schema, String fieldName,
Object fieldDefaultValue) {
ValidationUtils.checkArgument(fieldDefaultValue != null);
List<Schema.Field> filteredFields = schema.getFields()
@@ -494,7 +533,7 @@ public class HoodieAvroUtils {
/**
* Wraps schema as nullable if original was a nullable union.
*/
- public static Schema wrapNullable(Schema original, Schema updated) {
+ private static Schema wrapNullable(Schema original, Schema updated) {
if (original.getType() == Schema.Type.UNION) {
List<Schema> types = original.getTypes();
if (types.stream().anyMatch(s -> s.getType() == Schema.Type.NULL)) {
@@ -575,6 +614,7 @@ public class HoodieAvroUtils {
* <p>
* To better understand conversion rules please check {@link
#rewriteRecord(GenericRecord, Schema)}
*/
+ @VisibleForTesting
public static List<GenericRecord> rewriteRecords(List<GenericRecord>
records, Schema newSchema) {
return records.stream().map(r -> rewriteRecord(r,
newSchema)).collect(Collectors.toList());
}
@@ -698,7 +738,7 @@ public class HoodieAvroUtils {
* @return the string form of the field
* or empty if the schema does not contain the field name or the value is
null
*/
- public static Option<String> getNullableValAsString(GenericRecord rec,
String fieldName) {
+ private static Option<String> getNullableValAsString(GenericRecord rec,
String fieldName) {
Schema.Field field = rec.getSchema().getField(fieldName);
String fieldVal = field == null ? null :
StringUtils.objToString(rec.get(field.pos()));
return Option.ofNullable(fieldVal);
@@ -883,20 +923,22 @@ public class HoodieAvroUtils {
*
* @param record Hoodie record.
* @param columns Names of the columns to get values.
- * @param schema {@link Schema} instance.
+ * @param schema {@link HoodieSchema} instance.
* @return Column value.
*/
public static Object[] getRecordColumnValues(HoodieRecord record,
String[] columns,
- Schema schema,
+ HoodieSchema schema,
boolean
consistentLogicalTimestampEnabled) {
try {
- GenericRecord genericRecord = (GenericRecord)
(record.toIndexedRecord(HoodieAvroSchemaCache.intern(schema), new
Properties()).get()).getData();
- List<Object> list = new ArrayList<>();
- for (String col : columns) {
- list.add(HoodieAvroUtils.getNestedFieldVal(genericRecord, col, true,
consistentLogicalTimestampEnabled));
+ // Intern through the Avro-keyed cache so the per-comparison lookups in
RDDBucketIndexPartitioner's sort
+ // comparator and the per-record lookups in
HoodieTableMetadataUtil#collectColumnRangeMetadata stay O(1).
+ GenericRecord genericRecord = (GenericRecord)
(record.toIndexedRecord(HoodieAvroSchemaCache.intern(schema.toAvroSchema()),
PROPERTIES).get()).getData();
+ Object[] values = new Object[columns.length];
+ for (int i = 0; i < columns.length; i++) {
+ values[i] = HoodieAvroUtils.getNestedFieldVal(genericRecord,
columns[i], true, consistentLogicalTimestampEnabled);
}
- return list.toArray();
+ return values;
} catch (IOException e) {
throw new HoodieIOException("Unable to read record with key:" +
record.getKey(), e);
}
@@ -907,7 +949,7 @@ public class HoodieAvroUtils {
*
* @param record Hoodie record.
* @param columns Names of the columns to get values.
- * @param schema {@link Schema} instance.
+ * @param schema {@link HoodieSchema} instance.
* @return Column value.
*/
public static Object[]
getSortColumnValuesWithPartitionPathAndRecordKey(HoodieRecord record,
@@ -1108,6 +1150,7 @@ public class HoodieAvroUtils {
return result;
}
+ @VisibleForTesting
public static Object rewritePrimaryType(Object oldValue, Schema oldSchema,
Schema newSchema) {
// Normalize any java.time form (LocalDate / Instant / LocalDateTime) to
the Avro primitive
// form (Integer / Long) before doing any numeric / string conversion. The
legacy branches
@@ -1274,7 +1317,8 @@ public class HoodieAvroUtils {
* bytes is the result of BigDecimal.unscaledValue().toByteArray();
* This is also what Conversions.DecimalConversion.toBytes() outputs inside
a byte buffer
*/
- public static Object convertBytesToFixed(byte[] bytes, Schema schema) {
+ @VisibleForTesting
+ static Object convertBytesToFixed(byte[] bytes, Schema schema) {
LogicalTypes.Decimal decimal = (LogicalTypes.Decimal)
schema.getLogicalType();
BigDecimal bigDecimal = convertBytesToBigDecimal(bytes, decimal);
return DECIMAL_CONVERSION.toFixed(bigDecimal, schema, decimal);
@@ -1286,7 +1330,8 @@ public class HoodieAvroUtils {
* bytes is the result of BigDecimal.unscaledValue().toByteArray();
* This is also what Conversions.DecimalConversion.toBytes() outputs inside
a byte buffer
*/
- public static BigDecimal convertBytesToBigDecimal(byte[] value,
LogicalTypes.Decimal decimal) {
+ @VisibleForTesting
+ static BigDecimal convertBytesToBigDecimal(byte[] value,
LogicalTypes.Decimal decimal) {
return convertBytesToBigDecimal(value, decimal.getPrecision(),
decimal.getScale());
}
@@ -1295,6 +1340,23 @@ public class HoodieAvroUtils {
scale, new MathContext(precision, RoundingMode.HALF_UP));
}
+ /**
+ * Converts a byte array to a BigDecimal using the given decimal schema.
+ *
+ * @param value the byte array to convert
+ * @param decimalSchema the decimal schema containing precision and scale
+ * @return the resulting BigDecimal
+ * @throws IllegalArgumentException if the schema is not a DECIMAL type
+ */
+ public static BigDecimal convertBytesToBigDecimal(byte[] value, HoodieSchema
decimalSchema) {
+ ValidationUtils.checkArgument(decimalSchema != null, "Decimal schema
cannot be null");
+ ValidationUtils.checkArgument(decimalSchema.getType() ==
HoodieSchemaType.DECIMAL,
+ () -> "Schema must be of DECIMAL type, but is " +
decimalSchema.getType());
+
+ HoodieSchema.Decimal decimal = (HoodieSchema.Decimal) decimalSchema;
+ return convertBytesToBigDecimal(value, decimal.getPrecision(),
decimal.getScale());
+ }
+
/**
* Projects a record to a new schema by performing a shallow copy of fields.
* Best used for adding or removing top-level metadata fields.
@@ -1403,10 +1465,6 @@ public class HoodieAvroUtils {
/**
* convert days to Date
- * <p>
- * NOTE: This method could only be used in tests
- *
- * @VisibleForTesting
*/
public static java.sql.Date toJavaDate(int days) {
LocalDate date = LocalDate.ofEpochDay(days);
@@ -1417,10 +1475,6 @@ public class HoodieAvroUtils {
/**
* convert Date to days
- * <p>
- * NOTE: This method could only be used in tests
- *
- * @VisibleForTesting
*/
public static int fromJavaDate(Date date) {
long millisUtc = date.getTime();
@@ -1477,17 +1531,19 @@ public class HoodieAvroUtils {
/**
* Utility method to convert bytes to HoodieRecord using schema and payload
class.
*/
- public static <R> HoodieRecord<R> convertToRecord(GenericRecord rec, String
payloadClazz, String[] preCombineFields, boolean withOperationField) {
+ @VisibleForTesting
+ static <R> HoodieRecord<R> convertToRecord(GenericRecord rec, String
payloadClazz, String[] preCombineFields, boolean withOperationField) {
return convertToRecord(rec, payloadClazz, preCombineFields,
Pair.of(HoodieRecord.RECORD_KEY_METADATA_FIELD,
HoodieRecord.PARTITION_PATH_METADATA_FIELD),
withOperationField, Option.empty(), Option.empty());
}
- public static <R> HoodieRecord<R> convertToRecord(GenericRecord record,
String payloadClazz,
- String[]
preCombineFields,
- boolean
withOperationField,
-
Option<String> partitionName,
-
Option<HoodieSchema> schemaWithoutMetaFields) {
+ @VisibleForTesting
+ static <R> HoodieRecord<R> convertToRecord(GenericRecord record, String
payloadClazz,
+ String[]
preCombineFields,
+ boolean
withOperationField,
+ Option<String>
partitionName,
+ Option<HoodieSchema>
schemaWithoutMetaFields) {
return convertToRecord(record, payloadClazz, preCombineFields,
Pair.of(HoodieRecord.RECORD_KEY_METADATA_FIELD,
HoodieRecord.PARTITION_PATH_METADATA_FIELD),
withOperationField, partitionName, schemaWithoutMetaFields);
@@ -1496,6 +1552,7 @@ public class HoodieAvroUtils {
/**
* Utility method to convert bytes to HoodieRecord using schema and payload
class.
*/
+ @VisibleForTesting
public static <R> HoodieRecord<R> convertToRecord(GenericRecord record,
String payloadClazz,
String[]
preCombineFields,
Pair<String,
String> recordKeyPartitionPathFieldPair,
@@ -1581,15 +1638,17 @@ public class HoodieAvroUtils {
return rewriteRecordWithNewSchema(oldRecord, newSchema,
Collections.EMPTY_MAP, validate);
}
+ @VisibleForTesting
public static boolean gteqAvro1_9() {
return AVRO_VERSION != null && StringUtils.compareVersions(AVRO_VERSION,
"1.9") >= 0;
}
- public static boolean gteqAvro1_10() {
+ @VisibleForTesting
+ static boolean gteqAvro1_10() {
return AVRO_VERSION != null && StringUtils.compareVersions(AVRO_VERSION,
"1.10") >= 0;
}
- static boolean gteqAvro1_12() {
+ private static boolean gteqAvro1_12() {
return AVRO_VERSION != null && StringUtils.compareVersions(AVRO_VERSION,
"1.12") >= 0;
}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/avro/HoodieAvroWrapperUtils.java
b/hudi-common/src/main/java/org/apache/hudi/common/avro/HoodieAvroWrapperUtils.java
index e168e07b4582..6c7fbd041e8c 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/avro/HoodieAvroWrapperUtils.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/avro/HoodieAvroWrapperUtils.java
@@ -59,6 +59,16 @@ import static
org.apache.hudi.common.util.DateTimeUtils.instantToMicros;
import static org.apache.hudi.common.util.DateTimeUtils.microsToInstant;
import static
org.apache.hudi.metadata.HoodieTableMetadataUtil.tryUpcastDecimal;
+/**
+ * Wraps and unwraps Java values into the generated Avro {@code *Wrapper}
records used by column stats
+ * ({@code DateWrapper}, {@code DecimalWrapper}, {@code
TimestampMicrosWrapper}, and so on).
+ *
+ * <p>Value conversion only: nothing here manipulates schemas. The wrapper
schema is fixed by the generated
+ * record, and the only schema this class reads is the one it needs to pick
the right wrapper for a value.
+ * For schema work see {@link org.apache.hudi.common.schema.HoodieSchema} and
+ * {@link org.apache.hudi.common.schema.HoodieSchemaUtils}; for general record
and value operations see
+ * {@link HoodieAvroUtils}.</p>
+ */
public class HoodieAvroWrapperUtils {
private static final Conversions.DecimalConversion AVRO_DECIMAL_CONVERSION =
new Conversions.DecimalConversion();
@@ -201,7 +211,7 @@ public class HoodieAvroWrapperUtils {
}
}
- public static Comparable<?> unwrapAvroValueWrapper(Object avroValueWrapper,
String wrapperClassName) {
+ private static Comparable<?> unwrapAvroValueWrapper(Object avroValueWrapper,
String wrapperClassName) {
if (avroValueWrapper == null) {
return null;
} else if (DateWrapper.class.getSimpleName().equals(wrapperClassName)) {
@@ -301,7 +311,7 @@ public class HoodieAvroWrapperUtils {
return
BytesWrapper.newBuilder(BYTES_WRAPPER_BUILDER_STUB.get()).setValue((ByteBuffer)
value).build();
}
- public static Object wrapArray(Comparable<?> value, Function<Comparable<?>,
Object> wrapper) {
+ private static Object wrapArray(Comparable<?> value, Function<Comparable<?>,
Object> wrapper) {
List<Object> avroValues = OrderingValues.getValues((ArrayComparable)
value).stream().map(wrapper::apply).collect(Collectors.toList());
return
ArrayWrapper.newBuilder(ARRAY_WRAPPER_BUILDER_STUB.get()).setWrappedValues(avroValues).build();
}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/avro/processors/DecimalLogicalTypeProcessor.java
b/hudi-common/src/main/java/org/apache/hudi/common/avro/processors/DecimalLogicalTypeProcessor.java
index bfc3ec705b56..2544fbb95794 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/avro/processors/DecimalLogicalTypeProcessor.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/avro/processors/DecimalLogicalTypeProcessor.java
@@ -18,8 +18,8 @@
package org.apache.hudi.common.avro.processors;
+import org.apache.hudi.common.avro.HoodieAvroUtils;
import org.apache.hudi.common.schema.HoodieSchema;
-import org.apache.hudi.common.schema.HoodieSchemaUtils;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.avro.LogicalTypes;
@@ -62,7 +62,7 @@ public abstract class DecimalLogicalTypeProcessor extends
JsonFieldProcessor {
// Case 2: Object is a number in String format.
try {
//encoded big decimal
- bigDecimal =
HoodieSchemaUtils.convertBytesToBigDecimal(decodeStringToBigDecimalBytes(obj),
schema);
+ bigDecimal =
HoodieAvroUtils.convertBytesToBigDecimal(decodeStringToBigDecimalBytes(obj),
schema);
} catch (IllegalArgumentException e) {
//no-op
}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/engine/RecordContext.java
b/hudi-common/src/main/java/org/apache/hudi/common/engine/RecordContext.java
index f529bbed3ff2..01f01a861c18 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/engine/RecordContext.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/engine/RecordContext.java
@@ -24,11 +24,11 @@ import org.apache.hudi.common.model.DeleteRecord;
import org.apache.hudi.common.model.HoodieOperation;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.schema.HoodieSchema;
+import org.apache.hudi.common.schema.LocalHoodieSchemaCache;
import org.apache.hudi.common.table.HoodieTableConfig;
import org.apache.hudi.common.table.read.BufferedRecord;
import org.apache.hudi.common.table.read.DeleteContext;
import org.apache.hudi.common.util.JavaTypeConverter;
-import org.apache.hudi.common.util.LocalHoodieSchemaCache;
import org.apache.hudi.common.util.OrderingValues;
import org.apache.hudi.common.util.collection.ArrayComparable;
import org.apache.hudi.common.util.collection.Pair;
@@ -65,7 +65,7 @@ public abstract class RecordContext<T> implements
Serializable {
private final SerializableBiFunction<T, HoodieSchema, String>
recordKeyExtractor;
// for encoding and decoding schemas to the spillable map
- private final LocalHoodieSchemaCache localSchemaCache =
LocalHoodieSchemaCache.getInstance();
+ private final LocalHoodieSchemaCache localSchemaCache =
LocalHoodieSchemaCache.create();
@Getter
protected final JavaTypeConverter typeConverter;
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieAvroRecord.java
b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieAvroRecord.java
index 284fc655121e..cc8339947230 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieAvroRecord.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieAvroRecord.java
@@ -125,7 +125,7 @@ public class HoodieAvroRecord<T extends
HoodieRecordPayload> extends HoodieRecor
@Override
public Object[] getColumnValues(HoodieSchema recordSchema, String[] columns,
boolean consistentLogicalTimestampEnabled) {
- return HoodieAvroUtils.getRecordColumnValues(this, columns,
recordSchema.toAvroSchema(), consistentLogicalTimestampEnabled);
+ return HoodieAvroUtils.getRecordColumnValues(this, columns, recordSchema,
consistentLogicalTimestampEnabled);
}
@Override
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaCompatibilityChecker.java
b/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaCompatibilityChecker.java
index b54236578d4d..f92c043c8b45 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaCompatibilityChecker.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaCompatibilityChecker.java
@@ -65,7 +65,7 @@ public class HoodieSchemaCompatibilityChecker {
/**
* Message to annotate reader/writer schema pairs that are compatible.
*/
- public static final String READER_WRITER_COMPATIBLE_MESSAGE = "Reader schema
can always successfully decode data written using the writer schema.";
+ private static final String READER_WRITER_COMPATIBLE_MESSAGE = "Reader
schema can always successfully decode data written using the writer schema.";
/**
* Validates that the provided reader schema can be used to decode avro data
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaRepair.java
b/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaRepair.java
index bfe7e15c5eb3..c00470238d1c 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaRepair.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaRepair.java
@@ -24,6 +24,25 @@ import org.apache.hudi.common.util.Option;
import java.util.ArrayList;
import java.util.List;
+/**
+ * Repairs the logical types of a file schema using the table schema as the
reference.
+ *
+ * <p>Some writers persist a field without the logical type the table
declares, or with a coarser one: a
+ * bare {@code long} where the table says local-timestamp, or timestamp-micros
where the table says
+ * timestamp-millis. This class walks the file schema against the table schema
and restores the table's
+ * logical type wherever the two only differ that way, returning an interned
schema. When nothing needs
+ * repairing the input instance is returned as-is, so callers can compare by
reference.</p>
+ *
+ * <p>This is neither a compatibility check nor schema evolution: no field is
added, removed or promoted,
+ * and a genuine type mismatch is left alone. For reader/writer compatibility
use
+ * {@link HoodieSchemaCompatibility}; for evolution use
+ * {@code
org.apache.hudi.common.schema.internal.utils.AvroSchemaEvolutionUtils}.</p>
+ *
+ * <p>{@link #hasTimestampMillisField(HoodieSchema)} is the cheap pre-check
used to decide whether the
+ * repair is worth wiring in at all. Its sibling in the metadata-table domain
is
+ * {@code HoodieTableMetadataUtil#isTimestampMillisField}, which answers the
same question for one field
+ * schema rather than recursively for a whole table schema.</p>
+ */
public class HoodieSchemaRepair {
public static HoodieSchema repairLogicalTypes(HoodieSchema fileSchema,
HoodieSchema tableSchema) {
HoodieSchema repairedSchema = repairSchema(fileSchema, tableSchema);
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaTypePromotion.java
b/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaTypePromotion.java
index b6bdc27726eb..13ae75f194a6 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaTypePromotion.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaTypePromotion.java
@@ -19,22 +19,30 @@
package org.apache.hudi.common.schema;
/**
- * Defines type promotion rules for HoodieSchema compatibility checking.
+ * The single table of primitive widening promotions, used by {@link
HoodieSchemaProjectionChecker}.
*
- * <p>Type promotion allows a reader schema with a "wider" type to read data
- * written with a "narrower" type. This follows Avro's type promotion
rules.</p>
- *
- * <p>Supported promotions:
+ * <p>A promotion lets a reader schema with a wider type read data written
with a narrower one:</p>
* <ul>
- * <li>INT → LONG, FLOAT, DOUBLE</li>
- * <li>LONG → FLOAT, DOUBLE</li>
- * <li>FLOAT → DOUBLE</li>
- * <li>STRING ↔ BYTES (bidirectional)</li>
- * <li>Decimal precision widening: (p2-s2) ≥ (p1-s1) and s2 ≥ s1</li>
+ * <li>INT -> LONG, FLOAT, DOUBLE</li>
+ * <li>LONG -> FLOAT, DOUBLE</li>
+ * <li>FLOAT -> DOUBLE</li>
+ * <li>STRING <-> BYTES (bidirectional)</li>
+ * <li>STRING <- any numeric type</li>
+ * <li>decimal widening, see {@link #isDecimalWidening(HoodieSchema,
HoodieSchema)}</li>
* </ul>
- * </p>
*
- * <p>This class is package-private and used internally by schema
compatibility checkers.</p>
+ * <p>Logical-type-over-primitive promotions are deliberately NOT in this
table. A TIMESTAMP reader over a
+ * LONG writer, or a UUID reader over a STRING writer, is accepted by
+ * {@link HoodieSchemaCompatibilityChecker} for reader/writer compatibility,
but it must not make a bare
+ * long a "compatible projection" of a timestamp: writer-schema deduction
would then silently drop the
+ * logical type. Compatibility and projection are different questions, so they
use different tables.</p>
+ *
+ * <p>One more difference is documented rather than resolved:
+ * {@link #isDecimalWidening(HoodieSchema, HoodieSchema)} additionally
requires the same backing (fixed
+ * versus bytes) and, for fixed, an equal fixed size, whereas the decimal
check in
+ * {@code HoodieSchemaCompatibilityChecker} compares only precision and
scale.</p>
+ *
+ * <p>This class is package-private and used only by {@link
HoodieSchemaProjectionChecker}.</p>
*/
class HoodieSchemaTypePromotion {
@@ -44,7 +52,7 @@ class HoodieSchemaTypePromotion {
/**
* Checks if the reader type can be promoted from the writer type.
- * This allows type widening (e.g., int → long) but not narrowing.
+ * This allows type widening (e.g., int -> long) but not narrowing.
*
* @param readerType the type in the reader schema
* @param writerType the type in the writer schema
@@ -90,8 +98,8 @@ class HoodieSchemaTypePromotion {
* <ul>
* <li>Both schemas are decimals with the same underlying type (FIXED or
BYTES)</li>
* <li>Reader precision and scale are equal or wider than writer's</li>
- * <li>Specifically: (readerPrecision - readerScale) ≥ (writerPrecision -
writerScale)</li>
- * <li>And: readerScale ≥ writerScale</li>
+ * <li>Specifically: (readerPrecision - readerScale) >=
(writerPrecision - writerScale)</li>
+ * <li>And: readerScale >= writerScale</li>
* </ul>
* </p>
*
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaUtils.java
b/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaUtils.java
index 98a2d262a8fa..fea4bc3ae8fa 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaUtils.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaUtils.java
@@ -24,6 +24,7 @@ import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.schema.internal.HoodieSchemaException;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.ValidationUtils;
+import org.apache.hudi.common.util.VisibleForTesting;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieException;
@@ -31,27 +32,59 @@ import org.apache.avro.JsonProperties;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericData;
-import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
-import java.util.function.Function;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
+import java.util.stream.Stream;
/**
- * Utility class for HoodieSchema operations including table schema
manipulation,
- * compatibility checking, and schema evolution operations.
+ * HoodieSchema-typed structural transforms of table schemas and of the
well-known Hudi record shapes.
*
- * <p>This class provides HoodieSchema equivalents of operations found in
AvroSchemaUtils
- * and HoodieAvroUtils, focusing on table schema management rather than
record-level operations.</p>
+ * <p>What lives here:</p>
+ * <ul>
+ * <li>metadata fields: {@link #addMetadataFields(HoodieSchema, boolean)},
+ * {@link #removeMetadataFields(HoodieSchema)}, {@link
#createHoodieWriteSchema(String, boolean)},
+ * {@link #isMetadataField(String)}</li>
+ * <li>record-key and delete-log schemas: {@link #getRecordKeySchema()},
+ * {@link #getRecordKeyPartitionPathSchema()}, {@link
#createDeleteLogSchema(HoodieSchema, List)}</li>
+ * <li>projection and pruning: {@link
#generateProjectionSchema(HoodieSchema, List)},
+ * {@link #projectSchema(HoodieSchema, List)}, {@link
#pruneDataSchema(HoodieSchema, HoodieSchema, Set)},
+ * {@link #removeFields(HoodieSchema, Set)}</li>
+ * <li>appending and merging fields: {@link
#appendFieldsToSchema(HoodieSchema, List)},
+ * {@link #appendFieldsToSchemaDedupNested(HoodieSchema, List)},
+ * {@link #mergeSchemas(HoodieSchema, HoodieSchema)},
+ * {@link #createNewSchemaFromFieldsWithReference(HoodieSchema,
List)}</li>
+ * <li>field copies and defaults: {@link
#createNewSchemaField(HoodieSchemaField)} (the other
+ * {@code createNewSchemaField} overloads are validated aliases of
{@code HoodieSchemaField.of}),
+ * {@link #toJavaDefaultValue(HoodieSchemaField)}</li>
+ * <li>nullability: {@link #asNullable(HoodieSchema)}</li>
+ * <li>naming: {@link #sanitizeName(String)}, {@link
#getRecordQualifiedName(String)}</li>
+ * <li>lookups and predicates that need more than {@link HoodieSchema}
offers on its own:
+ * {@link #findNestedField(HoodieSchema, String)}, {@link
#findMissingFields(HoodieSchema, HoodieSchema)},
+ * {@link #resolveUnionSchema(HoodieSchema, String)}, {@link
#hasDecimalField(HoodieSchema)}</li>
+ * </ul>
*
- * <p>All methods in this class delegate to the corresponding Avro utilities
internally
- * while providing a clean HoodieSchema-based API. This approach ensures
consistency
- * with existing behavior while enabling migration to HoodieSchema types.</p>
+ * <p>Not here:</p>
+ * <ul>
+ * <li>value and record operations: {@link
org.apache.hudi.common.avro.HoodieAvroUtils}</li>
+ * <li>reader/writer compatibility checks: {@link
HoodieSchemaCompatibility}</li>
+ * <li>questions about a single schema: instance methods on {@link
HoodieSchema}.
+ * {@link #getFieldSchema(HoodieSchema, String)} and {@link
#getNestedField(HoodieSchema, String)} are
+ * argument-checking facades over {@link HoodieSchema#getField(String)}
and
+ * {@link HoodieSchema#getNestedField(String)}, not lookups of their
own</li>
+ * <li>the field-id InternalSchema (schema-on-read) domain: {@code
org.apache.hudi.common.schema.internal}</li>
+ * </ul>
+ *
+ * <p>A few methods here still delegate to Avro-typed implementations
+ * ({@link #asNullable(HoodieSchema)}, {@link
#createNullableSchema(HoodieSchema)},
+ * {@link #projectSchema(HoodieSchema, List)} and the 5-arg
+ * {@link #createNewSchemaField(String, HoodieSchema, String, Object,
HoodieFieldOrder)}). Those delegations
+ * are being retired under #16639; new methods must be implemented on
HoodieSchema directly.</p>
*
* @since 1.2.0
*/
@@ -110,8 +143,10 @@ public final class HoodieSchemaUtils {
}
/**
- * Adds Hudi metadata fields to the given schema.
- * This is equivalent to HoodieAvroUtils.addMetadataFields() but operates on
HoodieSchema.
+ * Prepends the Hudi metadata columns ({@code _hoodie_commit_time}, {@code
_hoodie_commit_seqno},
+ * {@code _hoodie_record_key}, {@code _hoodie_partition_path}, {@code
_hoodie_file_name}) to the given
+ * schema; {@code withOperationField} additionally adds {@code
_hoodie_operation}. Metadata columns
+ * already present on the input schema are dropped rather than duplicated.
*
* @param schema the input schema
* @param withOperationField whether to include operation metadata field
@@ -163,8 +198,7 @@ public final class HoodieSchemaUtils {
}
/**
- * Removes Hudi metadata fields from the given schema.
- * This is equivalent to HoodieAvroUtils.removeMetadataFields() but operates
on HoodieSchema.
+ * Removes the Hudi metadata columns, including {@code _hoodie_operation},
from the given schema.
*
* @param schema the input schema with metadata fields
* @return new HoodieSchema with metadata fields removed
@@ -182,6 +216,11 @@ public final class HoodieSchemaUtils {
/**
* Merges two schemas, combining fields from both with conflict resolution.
*
+ * <p>This is a plain recursive union of the two field lists: source field
order is preserved and
+ * target-only fields are appended. There is no type promotion and no
nullability reconciliation. For
+ * schema-evolution reconciliation use {@code
AvroSchemaEvolutionUtils#reconcileSchema} /
+ * {@code AvroSchemaEvolutionUtils#reconcileSchemaRequirements}.</p>
+ *
* @param sourceSchema source schema to merge from
* @param targetSchema target schema to merge into
* @return new HoodieSchema representing the merged result
@@ -209,8 +248,10 @@ public final class HoodieSchemaUtils {
}
/**
- * Creates a nullable version of the given schema (union with null).
- * This is equivalent to AvroSchemaUtils.createNullableSchema() but operates
on HoodieSchema.
+ * Creates a nullable version of the given schema (union of null and the
schema).
+ *
+ * <p>{@link HoodieSchema#createNullable(HoodieSchema)} is the idempotent
native equivalent and is
+ * preferred; this overload round-trips through Avro and is retained only
for existing call sites.</p>
*
* @param schema the input schema
* @return new HoodieSchema that allows null values
@@ -226,10 +267,10 @@ public final class HoodieSchemaUtils {
/**
* Create a new schema by force changing all the fields as nullable.
- * This is equivalent to AvroSchemaUtils.asNullable() but operates on
HoodieSchema.
*
* @return a new schema with all the fields updated as nullable
* @throws IllegalArgumentException if schema is null
+ * @see AvroSchemaUtils#asNullable(Schema)
*/
public static HoodieSchema asNullable(HoodieSchema schema) {
ValidationUtils.checkArgument(schema != null, "Schema cannot be null");
@@ -240,8 +281,8 @@ public final class HoodieSchemaUtils {
}
/**
- * Removes specified fields from a RECORD schema.
- * This is equivalent to HoodieAvroUtils.removeFields() but operates on
HoodieSchema.
+ * Removes the named top-level fields from a RECORD schema, preserving the
record's name, namespace,
+ * error flag and custom properties. Returns the input schema unchanged when
no field matches.
*
* @param schema original schema (must be RECORD type)
* @param fieldNamesToRemove set of field names to remove
@@ -286,28 +327,32 @@ public final class HoodieSchemaUtils {
}
/**
- * Finds fields that are present in the table schema but missing in the
writer schema.
- * This is equivalent to AvroSchemaUtils.findMissingFields() but operates on
HoodieSchemas.
+ * Finds the top-level fields that are present in the table schema but
missing from the writer schema.
+ * Nested records are not descended into; {@code
HoodieSchemaCompatibility#checkValidEvolution} is the
+ * one that finds missing fields recursively.
*
* @param tableSchema the complete table schema
* @param writerSchema the writer schema to check against
* @return list of HoodieSchemaFields that are missing in writer schema
* @throws IllegalArgumentException if either schema is null
+ * @see HoodieSchemaCompatibility#checkValidEvolution(HoodieSchema,
HoodieSchema)
*/
public static List<HoodieSchemaField> findMissingFields(HoodieSchema
tableSchema, HoodieSchema writerSchema) {
return findMissingFields(tableSchema, writerSchema,
Collections.emptySet());
}
/**
- * Finds fields that are present in the table schema but missing in the
writer schema,
- * excluding partition columns from the check.
- * This is equivalent to AvroSchemaUtils.findMissingFields() but operates on
HoodieSchemas.
+ * Finds the top-level fields that are present in the table schema but
missing from the writer schema,
+ * skipping the excluded column names (typically the partition columns).
Nested records are not descended
+ * into; {@code HoodieSchemaCompatibility#checkValidEvolution} is the one
that finds missing fields
+ * recursively.
*
* @param tableSchema the complete table schema
* @param writerSchema the writer schema to check against
* @param excludeColumns column names to exclude from missing field check
* @return list of HoodieSchemaFields that are missing in writer schema
* @throws IllegalArgumentException if either schema is null
+ * @see HoodieSchemaCompatibility#checkValidEvolution(HoodieSchema,
HoodieSchema)
*/
public static List<HoodieSchemaField> findMissingFields(HoodieSchema
tableSchema, HoodieSchema writerSchema,
Set<String>
excludeColumns) {
@@ -331,8 +376,8 @@ public final class HoodieSchemaUtils {
}
/**
- * Creates a new schema field with the specified properties.
- * This is equivalent to HoodieAvroUtils.createNewSchemaField() but returns
HoodieSchemaField.
+ * Alias of {@link HoodieSchemaField#of(String, HoodieSchema, String,
Object)} with argument validation.
+ * Prefer {@code HoodieSchemaField.of} directly in new code.
*
* @param name field name
* @param schema field schema
@@ -350,8 +395,10 @@ public final class HoodieSchemaUtils {
}
/**
- * Creates a new schema field with the specified properties, including field
order.
- * This is equivalent to HoodieAvroUtils.createNewSchemaField() but returns
HoodieSchemaField.
+ * Alias of {@link HoodieSchemaField#of(String, HoodieSchema, String,
Object, HoodieFieldOrder)} with
+ * argument validation. Prefer {@code HoodieSchemaField.of} directly in new
code; this overload still
+ * round-trips through the Avro-typed {@code
HoodieAvroUtils#createNewSchemaField} and is being retired
+ * under #16639.
*
* @param name field name
* @param schema field schema
@@ -375,8 +422,10 @@ public final class HoodieSchemaUtils {
}
/**
- * Creates a new HoodieSchemaField from an existing field.
- * This is equivalent to HoodieAvroUtils.createNewSchemaField() but returns
HoodieSchemaField.
+ * Copy factory: returns a new field carrying the same name, schema, doc and
default value as
+ * {@code field}. It exists because the backing Avro field cannot be shared
between two records, so a
+ * field taken off one schema has to be copied before being placed on
another. When building a field from
+ * scratch prefer {@link HoodieSchemaField#of(String, HoodieSchema, String,
Object)}.
*
* @param field the original HoodieSchemaField to create a new field from
* @return a new HoodieSchemaField with the same properties but properly
formatted default value
@@ -385,39 +434,15 @@ public final class HoodieSchemaUtils {
return createNewSchemaField(field.name(), field.schema(),
field.doc().orElse(null), field.defaultVal().orElse(null));
}
- /**
- * Converts a byte array to a BigDecimal using the given decimal schema.
- *
- * @param value the byte array to convert
- * @param decimalSchema the decimal schema containing precision and scale
- * @return the resulting BigDecimal
- * @throws IllegalArgumentException if the schema is not a DECIMAL type
- */
- public static BigDecimal convertBytesToBigDecimal(byte[] value, HoodieSchema
decimalSchema) {
- ValidationUtils.checkArgument(decimalSchema != null, "Decimal schema
cannot be null");
- ValidationUtils.checkArgument(decimalSchema.getType() ==
HoodieSchemaType.DECIMAL,
- () -> "Schema must be of DECIMAL type, but is " +
decimalSchema.getType());
-
- HoodieSchema.Decimal decimal = (HoodieSchema.Decimal) decimalSchema;
- return convertBytesToBigDecimal(value, decimal.getPrecision(),
decimal.getScale());
- }
-
- /**
- * Converts a byte array to a BigDecimal with the specified precision and
scale.
- * Delegates to {@link HoodieAvroUtils#convertBytesToBigDecimal(byte[], int,
int)}.
- *
- * @param value the byte array to convert
- * @param precision the precision of the decimal
- * @param scale the scale of the decimal
- * @return the resulting BigDecimal
- */
- public static BigDecimal convertBytesToBigDecimal(byte[] value, int
precision, int scale) {
- return HoodieAvroUtils.convertBytesToBigDecimal(value, precision, scale);
- }
-
/**
* Gets a field (including nested fields) from the schema using dot notation.
- * This method delegates to {@link HoodieSchema#getNestedField(String)}.
+ * This method is a null-checking facade over {@link
HoodieSchema#getNestedField(String)}: it returns the
+ * leaf field itself, paired with its canonical dotted path.
+ * <p>
+ * Not to be confused with {@link #findNestedField(HoodieSchema, String)},
which returns a synthesized
+ * lineage sub-schema rather than the leaf, and which does not understand
{@code list.element} /
+ * {@code key_value} path segments.
+ * </p>
* <p>
* Supports nested field access using dot notation. For example:
* <ul>
@@ -443,6 +468,12 @@ public final class HoodieSchemaUtils {
/**
* Generates a projection schema from the original schema, including only
the specified fields.
*
+ * <p>Field names are matched case-insensitively and the projected field
keeps the schema's original casing:
+ * Avro field names are case-sensitive while Hive lowercases column
projections before they reach the reader
+ * (see {@code HoodieRealtimeRecordReaderUtils#generateProjectionSchema}),
so both sides are lowercased for the
+ * lookup. A schema with two fields that differ only in case cannot be
projected and fails on the duplicate
+ * lowercase key.</p>
+ *
* @param originalSchema the source schema
* @param fieldNames the list of field names to include in the projection
* @return new HoodieSchema containing only the specified fields
@@ -606,6 +637,13 @@ public final class HoodieSchemaUtils {
/**
* Get gets a field from a record, works on nested fields as well (if you
provide the whole name, eg: toplevel.nextlevel.child)
+ * <p>
+ * Returns a synthesized lineage sub-schema, not the leaf field: {@code
b.z.z2} comes back as
+ * {@code b:record(z:record(z2))}. That shape is what {@link
#appendFieldsToSchemaDedupNested} consumes.
+ * Only record nesting is understood here - {@code list.element} and {@code
key_value} path segments are
+ * not. Use {@link #getNestedField(HoodieSchema, String)} or {@link
HoodieSchema#getNestedField(String)}
+ * when the leaf field and its canonical path are what is wanted.
+ * </p>
* @return the field, including its lineage.
* For example, if you have a schema: record(a:int, b:record(x:int, y:long,
z:record(z1: int, z2: float, z3: double), c:bool)
* "fieldName" | output
@@ -676,47 +714,11 @@ public final class HoodieSchemaUtils {
}
/**
- * Converts field values for specific data types with logical type handling.
- * This is equivalent to HoodieAvroUtils.convertValueForSpecificDataTypes()
but operates on HoodieSchema.
- * <p>
- * Handles special conversions for Avro logical types:
- * <ul>
- * <li>Date type - converts epoch day integer to LocalDate</li>
- * <li>Timestamp types - converts epoch milliseconds/microseconds to
Timestamp</li>
- * <li>Decimal type - converts bytes/fixed to BigDecimal</li>
- * </ul>
- *
- * @param fieldSchema the field schema
- * @param fieldValue the field value to convert
- * @param consistentLogicalTimestampEnabled whether to use consistent
logical timestamp handling
- * @return converted value for logical types, or original value
- * @throws IllegalStateException if fieldValue is null but schema is not
nullable
- * @since 1.2.0
- */
- public static Object convertValueForSpecificDataTypes(HoodieSchema
fieldSchema,
- Object fieldValue,
- boolean
consistentLogicalTimestampEnabled) {
- if (fieldSchema == null) {
- return fieldValue;
- } else if (fieldValue == null) {
- ValidationUtils.checkState(fieldSchema.isNullable(),
- "Field value is null but schema is not nullable");
- return null;
- }
-
- // Delegate to existing Avro utility
- return HoodieAvroUtils.convertValueForSpecificDataTypes(
- fieldSchema.toAvroSchema(),
- fieldValue,
- consistentLogicalTimestampEnabled
- );
- }
-
- /**
- * Fetch schema for record key and partition path.
- * This is equivalent to HoodieAvroUtils.getRecordKeyPartitionPathSchema()
but returns HoodieSchema.
+ * Builds the two-column {@code HoodieRecordKey} record holding {@code
_hoodie_record_key} and
+ * {@code _hoodie_partition_path}, both nullable strings.
*
* @return HoodieSchema containing record key and partition path fields
+ * @see #getRecordKeySchema()
*/
public static HoodieSchema getRecordKeyPartitionPathSchema() {
List<HoodieSchemaField> toBeAddedFields = new ArrayList<>(2);
@@ -731,11 +733,41 @@ public final class HoodieSchemaUtils {
return HoodieSchema.createRecord("HoodieRecordKey", "", "", false,
toBeAddedFields);
}
+ /**
+ * Schema of a native delete log record: the record key plus the ordering
fields, which are
+ * always nullable (see the comment in the body).
+ */
+ public static HoodieSchema createDeleteLogSchema(HoodieSchema tableSchema,
List<String> orderingFieldNames) {
+ // Native delete logs store only the record key plus optional ordering
values, so ordering fields in
+ // the delete-log schema must always be nullable even when the table
schema marks them required.
+ // A delete record such as HoodieEmptyRecord may carry
OrderingValues.getDefault() as an in-memory
+ // sentinel rather than a real field value. Persist NULL for that missing
value so readers can map it
+ // back to the default ordering without confusing it with a real business
value such as 0.
+ List<HoodieSchemaField> fields = Stream.concat(
+ Stream.of(createNewSchemaField(
+ HoodieRecord.RECORD_KEY_METADATA_FIELD,
HoodieSchema.create(HoodieSchemaType.STRING), null, null)),
+ orderingFieldNames.stream().map(orderingFieldName ->
tableSchema.getField(orderingFieldName)
+ .map(field -> createNewSchemaField(
+ field.name(), HoodieSchema.createNullable(field.schema()),
field.doc().orElse(null), HoodieSchema.NULL_VALUE))
+ .orElseThrow(() ->
+ new IllegalArgumentException("Ordering field " +
orderingFieldName + " not found in table schema"))))
+ .collect(Collectors.toList());
+ return HoodieSchema.createRecord("hudi_delete_log_record", null, null,
fields);
+ }
+
/**
* Fetches projected schema given list of fields to project. The field can
be nested in format `a.b.c` where a is
- * the top level field, b is at second level and so on.
+ * the top level field, b is at second level and so on. Field names are
matched case-sensitively.
* This is equivalent to {@link HoodieAvroUtils#projectSchema(Schema, List)}
but operates on HoodieSchema.
*
+ * <p>The two sibling projection helpers differ:</p>
+ * <ul>
+ * <li>{@link #generateProjectionSchema(HoodieSchema, List)} - top-level
fields only, matched
+ * case-insensitively</li>
+ * <li>{@link #pruneDataSchema(HoodieSchema, HoodieSchema, Set)} - prunes
to the shape of a required
+ * schema instead of to a list of names</li>
+ * </ul>
+ *
* @param fileSchema the original schema
* @param fields list of fields to project
* @return projected schema containing only specified fields
@@ -769,47 +801,27 @@ public final class HoodieSchemaUtils {
}
public static boolean hasDecimalField(HoodieSchema schema) {
- return hasDecimalWithCondition(schema, unused -> true);
- }
-
- /**
- * Checks whether the provided schema contains a decimal with a precision
less than or equal to 18,
- * which allows the decimal to be stored as int/long instead of a fixed size
byte array in
- * <a
href="https://github.com/apache/parquet-format/blob/master/LogicalTypes.md";>parquet
logical types</a>
- * @param schema the input schema to search
- * @return true if the schema contains a small precision decimal field and
false otherwise
- */
- public static boolean hasSmallPrecisionDecimalField(HoodieSchema schema) {
- return hasDecimalWithCondition(schema,
HoodieSchemaUtils::isSmallPrecisionDecimalField);
- }
-
- private static boolean hasDecimalWithCondition(HoodieSchema schema,
Function<HoodieSchema.Decimal, Boolean> condition) {
switch (schema.getType()) {
case RECORD:
for (HoodieSchemaField field : schema.getFields()) {
- if (hasDecimalWithCondition(field.schema(), condition)) {
+ if (hasDecimalField(field.schema())) {
return true;
}
}
return false;
case ARRAY:
- return hasDecimalWithCondition(schema.getElementType(), condition);
+ return hasDecimalField(schema.getElementType());
case MAP:
- return hasDecimalWithCondition(schema.getValueType(), condition);
+ return hasDecimalField(schema.getValueType());
case UNION:
- return hasDecimalWithCondition(schema.getNonNullType(), condition);
+ return hasDecimalField(schema.getNonNullType());
case DECIMAL:
- HoodieSchema.Decimal decimal = (HoodieSchema.Decimal) schema;
- return condition.apply(decimal);
+ return true;
default:
return false;
}
}
- private static boolean isSmallPrecisionDecimalField(HoodieSchema.Decimal
decimal) {
- return decimal.getPrecision() <= 18;
- }
-
/**
* Resolves a union schema by finding the schema matching the given full
name.
* Handles both simple nullable unions (null + non-null) and complex unions
with multiple types.
@@ -850,6 +862,7 @@ public final class HoodieSchemaUtils {
return nonNullType;
}
+ @VisibleForTesting
public static String addMetadataColumnTypes(String hiveColumnTypes) {
return "string,string,string,string,string," + hiveColumnTypes;
}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemas.java
b/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemas.java
deleted file mode 100644
index 917988dd8aa4..000000000000
--- a/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemas.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hudi.common.schema;
-
-import org.apache.hudi.common.model.HoodieRecord;
-
-import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import static
org.apache.hudi.common.schema.HoodieSchemaUtils.createNewSchemaField;
-
-/**
- * Factory class for {@link HoodieSchema}.
- */
-public class HoodieSchemas {
-
- public static HoodieSchema createDeleteLogSchema(HoodieSchema tableSchema,
List<String> orderingFieldNames) {
- // Native delete logs store only the record key plus optional ordering
values, so ordering fields in
- // the delete-log schema must always be nullable even when the table
schema marks them required.
- // A delete record such as HoodieEmptyRecord may carry
OrderingValues.getDefault() as an in-memory
- // sentinel rather than a real field value. Persist NULL for that missing
value so readers can map it
- // back to the default ordering without confusing it with a real business
value such as 0.
- List<HoodieSchemaField> fields = Stream.concat(
- Stream.of(createNewSchemaField(
- HoodieRecord.RECORD_KEY_METADATA_FIELD,
HoodieSchema.create(HoodieSchemaType.STRING), null, null)),
- orderingFieldNames.stream().map(orderingFieldName ->
tableSchema.getField(orderingFieldName)
- .map(field -> createNewSchemaField(
- field.name(), HoodieSchema.createNullable(field.schema()),
field.doc().orElse(null), HoodieSchema.NULL_VALUE))
- .orElseThrow(() ->
- new IllegalArgumentException("Ordering field " +
orderingFieldName + " not found in table schema"))))
- .collect(Collectors.toList());
- return HoodieSchema.createRecord("hudi_delete_log_record", null, null,
fields);
- }
-}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/util/LocalHoodieSchemaCache.java
b/hudi-common/src/main/java/org/apache/hudi/common/schema/LocalHoodieSchemaCache.java
similarity index 90%
rename from
hudi-common/src/main/java/org/apache/hudi/common/util/LocalHoodieSchemaCache.java
rename to
hudi-common/src/main/java/org/apache/hudi/common/schema/LocalHoodieSchemaCache.java
index 6a11d88e7718..2012ca890c49 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/util/LocalHoodieSchemaCache.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/schema/LocalHoodieSchemaCache.java
@@ -16,9 +16,9 @@
* limitations under the License.
*/
-package org.apache.hudi.common.util;
+package org.apache.hudi.common.schema;
-import org.apache.hudi.common.schema.HoodieSchema;
+import org.apache.hudi.common.util.Option;
import javax.annotation.concurrent.NotThreadSafe;
@@ -46,7 +46,10 @@ public class LocalHoodieSchemaCache implements Serializable {
this.schemaToVersionId = new HashMap<>();
}
- public static LocalHoodieSchemaCache getInstance() {
+ /**
+ * Returns a new, empty cache. Each caller owns its own version-id space.
+ */
+ public static LocalHoodieSchemaCache create() {
return new LocalHoodieSchemaCache();
}
@@ -63,4 +66,4 @@ public class LocalHoodieSchemaCache implements Serializable {
public Option<HoodieSchema> getSchema(Integer versionId) {
return Option.ofNullable(this.versionIdToSchema.get(versionId));
}
-}
\ No newline at end of file
+}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/schema/internal/convert/InternalSchemaConverter.java
b/hudi-common/src/main/java/org/apache/hudi/common/schema/internal/convert/InternalSchemaConverter.java
index 69c97e7364c6..7b39c934e28e 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/schema/internal/convert/InternalSchemaConverter.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/schema/internal/convert/InternalSchemaConverter.java
@@ -173,7 +173,7 @@ public class InternalSchemaConverter {
* if we compare a schema that has not been converted to internal schema
* at any stage, the difference in ordering can cause issues. To resolve
this,
* we order null to be first for any HoodieSchema that enters into hudi.
- * AvroSchemaUtils.isProjectionOfInternal uses index based comparison for
unions.
+ * {@code HoodieSchemaProjectionChecker}'s isProjectionOfInternal uses index
based comparison for unions.
* Spark and flink don't support complex unions so this would not be an issue
* but for the metadata table HoodieMetadata.avsc uses a trick where we have
a bunch of
* different types wrapped in record for col stats.
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieNativeLogDeleteBlock.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieNativeLogDeleteBlock.java
index 031bbe609929..8dee4a27471d 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieNativeLogDeleteBlock.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieNativeLogDeleteBlock.java
@@ -26,7 +26,7 @@ import org.apache.hudi.common.model.HoodieFileFormat;
import org.apache.hudi.common.model.HoodieLogFile;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.schema.HoodieSchema;
-import org.apache.hudi.common.schema.HoodieSchemas;
+import org.apache.hudi.common.schema.HoodieSchemaUtils;
import org.apache.hudi.common.table.read.BufferedRecord;
import org.apache.hudi.common.table.read.BufferedRecords;
import org.apache.hudi.common.util.Option;
@@ -69,7 +69,7 @@ public class HoodieNativeLogDeleteBlock extends
HoodieDeleteBlock {
super(Option.empty(), null, true, getContentLocation(storage, logFile),
header, footer);
this.storage = storage;
this.logFile = logFile;
- this.deleteLogSchema =
HoodieSchemas.createDeleteLogSchema(getSchemaFromHeader(), orderingFieldNames);
+ this.deleteLogSchema =
HoodieSchemaUtils.createDeleteLogSchema(getSchemaFromHeader(),
orderingFieldNames);
this.orderingFieldNames = orderingFieldNames;
this.partitionPath = partitionPath;
this.props = props == null ? new Properties() : props;
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/read/lsm/LsmFileIterators.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/read/lsm/LsmFileIterators.java
index 4a43f1671948..1f1f1fa88625 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/read/lsm/LsmFileIterators.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/read/lsm/LsmFileIterators.java
@@ -25,7 +25,7 @@ import org.apache.hudi.common.model.HoodieBaseFile;
import org.apache.hudi.common.model.HoodieLogFile;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.schema.HoodieSchema;
-import org.apache.hudi.common.schema.HoodieSchemas;
+import org.apache.hudi.common.schema.HoodieSchemaUtils;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.TableSchemaResolver;
import org.apache.hudi.common.table.read.BufferedRecord;
@@ -173,7 +173,7 @@ final class LsmFileIterators {
StoragePath storagePath,
long fileSize,
List<String> orderingFieldNames) throws IOException {
- HoodieSchema deleteLogSchema = HoodieSchemas.createDeleteLogSchema(
+ HoodieSchema deleteLogSchema = HoodieSchemaUtils.createDeleteLogSchema(
readerContext.getSchemaHandler().getTableSchema(), orderingFieldNames);
ClosableIterator<T> recordIterator = createFileRecordIterator(
readerContext, storage, pathInfo, storagePath, fileSize,
deleteLogSchema, deleteLogSchema);
diff --git
a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataPayload.java
b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataPayload.java
index 810cbecacc03..6f100d65643e 100644
---
a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataPayload.java
+++
b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataPayload.java
@@ -24,7 +24,6 @@ import org.apache.hudi.avro.model.HoodieMetadataFileInfo;
import org.apache.hudi.avro.model.HoodieMetadataRecord;
import org.apache.hudi.avro.model.HoodieRecordIndexInfo;
import org.apache.hudi.avro.model.HoodieSecondaryIndexInfo;
-import org.apache.hudi.common.avro.AvroSchemaCache;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.EmptyHoodieRecordPayload;
import org.apache.hudi.common.model.HoodieAvroRecord;
@@ -109,8 +108,9 @@ public class HoodieMetadataPayload implements
HoodieRecordPayload<HoodieMetadata
// Note: Variable is unused, but caching is required.
private static final HoodieSchema HOODIE_METADATA_SCHEMA =
HoodieSchemaCache.intern(
HoodieSchema.fromAvroSchema(HoodieMetadataRecord.getClassSchema()));
- // Cache the Avro schema reference for O(1) equality checks during
Avro.Schema -> HoodieSchema migration
- private static final Schema HOODIE_METADATA_AVRO_SCHEMA =
AvroSchemaCache.intern(HoodieMetadataRecord.getClassSchema());
+ // Held for the reference-equality fast path in getInsertValue:
getClassSchema() returns the
+ // generated class's SCHEMA$ singleton, so no interning is needed.
+ private static final Schema HOODIE_METADATA_AVRO_SCHEMA =
HoodieMetadataRecord.getClassSchema();
/**
* Field offsets when metadata fields are present
*/
@@ -425,7 +425,7 @@ public class HoodieMetadataPayload implements
HoodieRecordPayload<HoodieMetadata
}
// TODO: feature(schema): Swap this over to HOODIE_METADATA_SCHEMA after
HoodieRecordPayload implementations are using HoodieSchema
- // Uses cached Avro schema reference for O(1) equality check.
+ // Reference equality against the generated class's SCHEMA$ singleton held
in HOODIE_METADATA_AVRO_SCHEMA.
if (schema == null || schema == HOODIE_METADATA_AVRO_SCHEMA) {
// If the schema is same or none is provided, we can return the record
directly
HoodieMetadataRecord record = new HoodieMetadataRecord(key, type,
filesystemMetadata, bloomFilterMetadata,
diff --git
a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java
b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java
index 202a2e6a0b33..6c87d584765a 100644
---
a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java
+++
b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java
@@ -328,7 +328,7 @@ public class HoodieTableMetadataUtil {
Object fieldValue;
HoodieSchemaType fieldSchemaType = fieldSchema.getType();
if (record.getRecordType() == HoodieRecordType.AVRO) {
- fieldValue = HoodieAvroUtils.getRecordColumnValues(record, new
String[]{fieldName}, recordSchema.toAvroSchema(), false)[0];
+ fieldValue = HoodieAvroUtils.getRecordColumnValues(record, new
String[]{fieldName}, recordSchema, false)[0];
if (fieldValue != null && fieldSchemaType.equals(HoodieSchemaType.DATE))
{
fieldValue = java.sql.Date.valueOf(fieldValue.toString());
}
diff --git
a/hudi-common/src/test/java/org/apache/hudi/common/avro/TestHoodieAvroUtils.java
b/hudi-common/src/test/java/org/apache/hudi/common/avro/TestHoodieAvroUtils.java
index 30c726c094a8..d59171dd653a 100644
---
a/hudi-common/src/test/java/org/apache/hudi/common/avro/TestHoodieAvroUtils.java
+++
b/hudi-common/src/test/java/org/apache/hudi/common/avro/TestHoodieAvroUtils.java
@@ -63,7 +63,9 @@ import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecordPayload;
import org.apache.hudi.common.model.OverwriteWithLatestAvroPayload;
import org.apache.hudi.common.model.RewriteAvroPayload;
+import org.apache.hudi.common.schema.HoodieAvroSchemaCache;
import org.apache.hudi.common.schema.HoodieSchema;
+import org.apache.hudi.common.schema.HoodieSchemaType;
import org.apache.hudi.common.schema.HoodieSchemaUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.exception.HoodieException;
@@ -124,6 +126,7 @@ import static
org.apache.hudi.common.schema.HoodieSchemaUtils.sanitizeName;
import static org.apache.hudi.common.util.StringUtils.getUTF8Bytes;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -566,6 +569,124 @@ public class TestHoodieAvroUtils {
assertEquals(FIXTURE_EPOCH_MICROS,
HoodieAvroUtils.convertValueForAvroLogicalTypes(LOCAL_TS_MICROS_SCHEMA,
FIXTURE_LOCAL_DT_MICROS, false));
}
+ @Test
+ public void testConvertValueForSpecificDataTypes_NullSchema() {
+ // Test with null schema - should return value unchanged
+ String testValue = "test_value";
+ Object result = HoodieAvroUtils.convertValueForSpecificDataTypes(null,
testValue, false);
+ assertEquals(testValue, result);
+ }
+
+ @Test
+ public void testConvertValueForSpecificDataTypes_NullValue_NullableSchema() {
+ // Test with null value and nullable schema - should return null
+ Schema nullableIntSchema =
HoodieSchema.createNullable(HoodieSchema.create(HoodieSchemaType.INT)).toAvroSchema();
+ Object result =
HoodieAvroUtils.convertValueForSpecificDataTypes(nullableIntSchema, null,
false);
+ assertNull(result);
+ }
+
+ @Test
+ public void
testConvertValueForSpecificDataTypes_NullValue_NonNullableSchema() {
+ // Test with null value and non-nullable schema - should throw exception
+ Schema nonNullableSchema = Schema.create(Schema.Type.STRING);
+ assertThrows(IllegalStateException.class, () ->
+ HoodieAvroUtils.convertValueForSpecificDataTypes(nonNullableSchema,
null, false));
+ }
+
+ @Test
+ public void testConvertValueForSpecificDataTypes_DateLogicalType() {
+ // Test value: epoch days for 2023-01-01
+ int epochDays = 19358;
+ Object result =
HoodieAvroUtils.convertValueForSpecificDataTypes(DATE_SCHEMA, epochDays, false);
+ assertNotNull(result);
+ assertTrue(result instanceof LocalDate);
+ assertEquals(LocalDate.of(2023, 1, 1), result);
+ }
+
+ @Test
+ public void testConvertValueForSpecificDataTypes_TimestampMillis_Enabled() {
+ // Test value: milliseconds for 2023-01-01 00:00:00
+ long millis = 1672560000000L;
+ Object result =
HoodieAvroUtils.convertValueForSpecificDataTypes(TS_MILLIS_SCHEMA, millis,
true);
+ assertNotNull(result);
+ assertTrue(result instanceof Timestamp);
+ assertEquals(new Timestamp(millis), result);
+ }
+
+ @Test
+ public void testConvertValueForSpecificDataTypes_TimestampMillis_Disabled() {
+ long millis = 1672560000000L;
+ Object result =
HoodieAvroUtils.convertValueForSpecificDataTypes(TS_MILLIS_SCHEMA, millis,
false);
+ assertEquals(millis, result);
+ }
+
+ @Test
+ public void testConvertValueForSpecificDataTypes_TimestampMicros_Enabled() {
+ // Test value: microseconds for 2023-01-01 00:00:00
+ long micros = 1672560000000000L;
+ Object result =
HoodieAvroUtils.convertValueForSpecificDataTypes(TS_MICROS_SCHEMA, micros,
true);
+ assertNotNull(result);
+ assertTrue(result instanceof Timestamp);
+ assertEquals(new Timestamp(micros / 1000), result);
+ }
+
+ @Test
+ public void testConvertValueForSpecificDataTypes_DecimalBytes() {
+ // Create decimal schema with precision=10, scale=2
+ Schema decimalSchema = HoodieSchema.createDecimal(10, 2).toAvroSchema();
+
+ // Create test value: 1234.56
+ BigDecimal expectedDecimal = new BigDecimal("1234.56");
+ ByteBuffer byteBuffer =
ByteBuffer.wrap(expectedDecimal.unscaledValue().toByteArray());
+ Object result =
HoodieAvroUtils.convertValueForSpecificDataTypes(decimalSchema, byteBuffer,
false);
+ assertNotNull(result);
+ assertTrue(result instanceof BigDecimal);
+ assertEquals(expectedDecimal, result);
+ }
+
+ @Test
+ public void testConvertValueForSpecificDataTypes_NonLogicalType() {
+ // Test with non-logical type (plain string) - should return unchanged
+ Schema stringSchema = Schema.create(Schema.Type.STRING);
+ String testValue = "test_string";
+ Object result =
HoodieAvroUtils.convertValueForSpecificDataTypes(stringSchema, testValue,
false);
+ assertEquals(testValue, result);
+ }
+
+ @Test
+ public void testConvertValueForSpecificDataTypes_UnionWithNull() {
+ // Test with union type containing null
+ Schema nullableDateSchema =
HoodieSchema.createNullable(HoodieSchema.createDate()).toAvroSchema();
+
+ // Test with non-null value
+ int epochDays = 19358; // 2023-01-01
+ Object result =
HoodieAvroUtils.convertValueForSpecificDataTypes(nullableDateSchema, epochDays,
false);
+ assertNotNull(result);
+ assertTrue(result instanceof LocalDate);
+ assertEquals(LocalDate.of(2023, 1, 1), result);
+ }
+
+ @Test
+ public void testConvertBytesToBigDecimalWithHoodieSchema() {
+ HoodieSchema decimalSchema = HoodieSchema.createDecimal(10, 2);
+ BigDecimal expected = new BigDecimal("1234.56");
+ assertEquals(expected,
+
HoodieAvroUtils.convertBytesToBigDecimal(expected.unscaledValue().toByteArray(),
decimalSchema));
+ }
+
+ @Test
+ public void testConvertBytesToBigDecimalWithNonDecimalHoodieSchema() {
+ HoodieSchema stringSchema = HoodieSchema.create(HoodieSchemaType.STRING);
+ assertThrows(IllegalArgumentException.class, () ->
+ HoodieAvroUtils.convertBytesToBigDecimal(new byte[] {0x01},
stringSchema));
+ }
+
+ @Test
+ public void testConvertBytesToBigDecimalWithNullHoodieSchema() {
+ assertThrows(IllegalArgumentException.class, () ->
+ HoodieAvroUtils.convertBytesToBigDecimal(new byte[] {0x01},
(HoodieSchema) null));
+ }
+
/**
* Cross-Avro-version invariant for ordering-value extraction: a record
whose timestamp/date field
* holds the java.time form (Avro 1.12.1 fast reader) must yield the same
comparable ordering value
@@ -955,6 +1076,49 @@ public class TestHoodieAvroUtils {
}
}
+ @Test
+ void testGetRecordColumnValues() {
+ Schema schema = new Schema.Parser().parse(SCHEMA_WITH_NESTED_FIELD_STR);
+ GenericRecord student = new
GenericData.Record(schema.getField("student").schema());
+ student.put("lastnameNested", "nested-last");
+ GenericRecord record = new GenericData.Record(schema);
+ record.put("firstname", "first");
+ record.put("lastname", "last");
+ record.put("student", student);
+ HoodieRecordPayload avroPayload = new RewriteAvroPayload(record);
+ HoodieAvroRecord avroRecord = new HoodieAvroRecord(new
HoodieKey("record1", "partition1"), avroPayload);
+ HoodieSchema hoodieSchema =
HoodieSchema.parse(SCHEMA_WITH_NESTED_FIELD_STR);
+
+ // A nested column resolves through the intermediate record; a missing
column yields null rather than
+ // throwing because getRecordColumnValues hardcodes returnNullIfNotFound.
+ assertArrayEquals(new Object[] {"first", "nested-last", null},
HoodieAvroUtils.getRecordColumnValues(
+ avroRecord, new String[] {"firstname", "student.lastnameNested",
"missing_col"}, hoodieSchema, false));
+
+ // A null intermediate record and a non-record intermediate both yield
null instead of throwing.
+ record.put("student", null);
+ assertArrayEquals(new Object[] {null, null},
HoodieAvroUtils.getRecordColumnValues(
+ avroRecord, new String[] {"student.lastnameNested",
"firstname.nested"}, hoodieSchema, false));
+ }
+
+ @Test
+ void testGetRecordColumnValuesInternsSchema() {
+ HoodieSchema interned = HoodieAvroSchemaCache.intern(new
Schema.Parser().parse(EXAMPLE_SCHEMA));
+ GenericRecord record = new GenericData.Record(interned.toAvroSchema());
+ record.put("timestamp", 3.5);
+ record.put("_row_key", "record1");
+ record.put("non_pii_col", "val1");
+ record.put("pii_col", "val2");
+ HoodieAvroRecord avroRecord = new HoodieAvroRecord(
+ new HoodieKey("record1", "partition1"), new
OverwriteWithLatestAvroPayload(record, 0));
+
+ // A freshly parsed, equal-but-distinct schema must resolve to the
interned instance so the payload hands back
+ // the record it holds; without the intern it re-serializes and the string
comes back as Utf8, not String.
+ Object[] columnValues = HoodieAvroUtils.getRecordColumnValues(
+ avroRecord, new String[] {"non_pii_col"},
HoodieSchema.parse(EXAMPLE_SCHEMA), false);
+ assertInstanceOf(String.class, columnValues[0]);
+ assertEquals("val1", columnValues[0]);
+ }
+
private static Stream<Arguments>
recordNeedsRewriteForExtendedAvroTypePromotion() {
Schema decimal1 = LogicalTypes.decimal(12,
2).addToSchema(Schema.create(Schema.Type.BYTES));
Schema decimal2 = LogicalTypes.decimal(10,
2).addToSchema(Schema.create(Schema.Type.BYTES));
diff --git
a/hudi-common/src/test/java/org/apache/hudi/common/schema/TestHoodieSchemaUtils.java
b/hudi-common/src/test/java/org/apache/hudi/common/schema/TestHoodieSchemaUtils.java
index d2eee99d31cd..1862f2c80a33 100644
---
a/hudi-common/src/test/java/org/apache/hudi/common/schema/TestHoodieSchemaUtils.java
+++
b/hudi-common/src/test/java/org/apache/hudi/common/schema/TestHoodieSchemaUtils.java
@@ -31,11 +31,7 @@ import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
-import java.math.BigDecimal;
import java.math.BigInteger;
-import java.nio.ByteBuffer;
-import java.sql.Timestamp;
-import java.time.LocalDate;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
@@ -1047,6 +1043,11 @@ public class TestHoodieSchemaUtils {
assertTrue(fieldNames1.contains("_row_key"));
assertTrue(fieldNames1.contains("timestamp"));
+ // Field names are matched case-insensitively; HiveHoodieReaderContext
lowercases names before calling this.
+ HoodieSchema schema2 =
HoodieSchemaUtils.generateProjectionSchema(originalSchema,
Arrays.asList("_ROW_KEY"));
+ assertEquals(1, schema2.getFields().size());
+ assertEquals("_row_key", schema2.getFields().get(0).name());
+
Throwable caughtException = assertThrows(HoodieException.class, () ->
HoodieSchemaUtils.generateProjectionSchema(originalSchema,
Arrays.asList("_row_key", "timestamp", "fake_field")));
assertTrue(caughtException.getMessage().contains("Field fake_field not
found in log schema. Query cannot proceed!"));
@@ -1306,115 +1307,6 @@ public class TestHoodieSchemaUtils {
assertTrue(HoodieSchemaCompatibility.isSchemaCompatible(projectedSchema,
expectedSchema, false));
}
- @Test
- public void testConvertValueForSpecificDataTypes_NullSchema() {
- // Test with null schema - should return value unchanged
- String testValue = "test_value";
- Object result = HoodieSchemaUtils.convertValueForSpecificDataTypes(null,
testValue, false);
- assertEquals(testValue, result);
- }
-
- @Test
- public void testConvertValueForSpecificDataTypes_NullValue_NullableSchema() {
- // Test with null value and nullable schema - should return null
- HoodieSchema nullableIntSchema =
HoodieSchema.createNullable(HoodieSchema.create(HoodieSchemaType.INT));
- Object result =
HoodieSchemaUtils.convertValueForSpecificDataTypes(nullableIntSchema, null,
false);
- assertNull(result);
- }
-
- @Test
- public void
testConvertValueForSpecificDataTypes_NullValue_NonNullableSchema() {
- // Test with null value and non-nullable schema - should throw exception
- HoodieSchema nonNullableSchema =
HoodieSchema.create(HoodieSchemaType.STRING);
- assertThrows(IllegalStateException.class, () ->
- HoodieSchemaUtils.convertValueForSpecificDataTypes(nonNullableSchema,
null, false));
- }
-
- @Test
- public void testConvertValueForSpecificDataTypes_DateLogicalType() {
- // Create date schema
- HoodieSchema dateSchema = HoodieSchema.createDate();
-
- // Test value: epoch days for 2023-01-01
- int epochDays = 19358;
- Object result =
HoodieSchemaUtils.convertValueForSpecificDataTypes(dateSchema, epochDays,
false);
- assertNotNull(result);
- assertTrue(result instanceof LocalDate);
- assertEquals(LocalDate.of(2023, 1, 1), result);
- }
-
- @Test
- public void testConvertValueForSpecificDataTypes_TimestampMillis_Enabled() {
- // Create timestamp-millis schema
- HoodieSchema timestampMillisSchema = HoodieSchema.createTimestampMillis();
-
- // Test value: milliseconds for 2023-01-01 00:00:00
- long millis = 1672560000000L;
- Object result =
HoodieSchemaUtils.convertValueForSpecificDataTypes(timestampMillisSchema,
millis, true);
- assertNotNull(result);
- assertTrue(result instanceof Timestamp);
- assertEquals(new Timestamp(millis), result);
- }
-
- @Test
- public void testConvertValueForSpecificDataTypes_TimestampMillis_Disabled() {
- // Create timestamp-millis schema
- HoodieSchema timestampMillisSchema = HoodieSchema.createTimestampMillis();
- long millis = 1672560000000L;
- Object result =
HoodieSchemaUtils.convertValueForSpecificDataTypes(timestampMillisSchema,
millis, false);
- assertEquals(millis, result);
- }
-
- @Test
- public void testConvertValueForSpecificDataTypes_TimestampMicros_Enabled() {
- // Create timestamp-micros schema
- HoodieSchema timestampMicrosSchema = HoodieSchema.createTimestampMicros();
-
- // Test value: microseconds for 2023-01-01 00:00:00
- long micros = 1672560000000000L;
- Object result =
HoodieSchemaUtils.convertValueForSpecificDataTypes(timestampMicrosSchema,
micros, true);
- assertNotNull(result);
- assertTrue(result instanceof Timestamp);
- assertEquals(new Timestamp(micros / 1000), result);
- }
-
- @Test
- public void testConvertValueForSpecificDataTypes_DecimalBytes() {
- // Create decimal schema with precision=10, scale=2
- HoodieSchema decimalSchema = HoodieSchema.createDecimal(10, 2);
-
- // Create test value: 1234.56
- BigDecimal expectedDecimal = new BigDecimal("1234.56");
- ByteBuffer byteBuffer =
ByteBuffer.wrap(expectedDecimal.unscaledValue().toByteArray());
- Object result =
HoodieSchemaUtils.convertValueForSpecificDataTypes(decimalSchema, byteBuffer,
false);
- assertNotNull(result);
- assertTrue(result instanceof BigDecimal);
- assertEquals(expectedDecimal, result);
- }
-
- @Test
- public void testConvertValueForSpecificDataTypes_NonLogicalType() {
- // Test with non-logical type (plain string) - should return unchanged
- HoodieSchema stringSchema = HoodieSchema.create(HoodieSchemaType.STRING);
- String testValue = "test_string";
- Object result =
HoodieSchemaUtils.convertValueForSpecificDataTypes(stringSchema, testValue,
false);
- assertEquals(testValue, result);
- }
-
- @Test
- public void testConvertValueForSpecificDataTypes_UnionWithNull() {
- // Test with union type containing null
- HoodieSchema dateSchema = HoodieSchema.createDate();
- HoodieSchema nullableDateSchema = HoodieSchema.createNullable(dateSchema);
-
- // Test with non-null value
- int epochDays = 19358; // 2023-01-01
- Object result =
HoodieSchemaUtils.convertValueForSpecificDataTypes(nullableDateSchema,
epochDays, false);
- assertNotNull(result);
- assertTrue(result instanceof LocalDate);
- assertEquals(LocalDate.of(2023, 1, 1), result);
- }
-
@Test
void testHasDecimalField() {
assertTrue(HoodieSchemaUtils.hasDecimalField(HoodieSchema.parse(SCHEMA_WITH_DECIMAL_FIELD)));
@@ -1442,13 +1334,6 @@ public class TestHoodieSchemaUtils {
assertTrue(HoodieSchemaUtils.hasDecimalField(recordWithMapAndDecArray));
}
- @Test
- void testHasSmallPrecisionDecimalField() {
-
assertTrue(HoodieSchemaUtils.hasSmallPrecisionDecimalField(HoodieSchema.parse(SCHEMA_WITH_DECIMAL_FIELD)));
-
assertFalse(HoodieSchemaUtils.hasSmallPrecisionDecimalField(HoodieSchema.parse(SCHEMA_WITH_AVRO_TYPES_STR)));
-
assertFalse(HoodieSchemaUtils.hasSmallPrecisionDecimalField(HoodieSchema.parse(EXAMPLE_SCHEMA)));
- }
-
@Test
void testResolveUnionSchemaWithNonUnionSchema() {
// Non-union schemas should be returned as-is
@@ -2198,4 +2083,90 @@ public class TestHoodieSchemaUtils {
assertEquals("value", result.get().getRight().name());
assertEquals(HoodieSchemaType.LONG,
result.get().getRight().schema().getType());
}
+
+ private static HoodieSchema deleteLogTableSchema() {
+ return HoodieSchema.createRecord(
+ "TestRecord",
+ null,
+ null,
+ Arrays.asList(
+ HoodieSchemaField.of("ts",
HoodieSchema.create(HoodieSchemaType.LONG), "ordering field doc", null),
+ HoodieSchemaField.of("name",
HoodieSchema.create(HoodieSchemaType.STRING)),
+ HoodieSchemaField.of("seq",
HoodieSchema.create(HoodieSchemaType.STRING)),
+ HoodieSchemaField.of("opt_ts", HoodieSchema.createUnion(
+ HoodieSchema.create(HoodieSchemaType.LONG),
HoodieSchema.create(HoodieSchemaType.NULL))),
+ HoodieSchemaField.of("ts_ms",
HoodieSchema.createTimestampMillis()),
+ HoodieSchemaField.of("amount", HoodieSchema.createDecimal(10, 2))
+ )
+ );
+ }
+
+ @Test
+ public void testCreateDeleteLogSchema() {
+ HoodieSchema deleteLogSchema =
+ HoodieSchemaUtils.createDeleteLogSchema(deleteLogTableSchema(),
Collections.singletonList("ts"));
+
+ assertEquals("hudi_delete_log_record", deleteLogSchema.getName());
+ assertEquals(2, deleteLogSchema.getFields().size());
+
+ // The record key is always present and never nullable.
+ HoodieSchemaField recordKeyField = deleteLogSchema.getFields().get(0);
+ assertEquals(HoodieRecord.RECORD_KEY_METADATA_FIELD,
recordKeyField.name());
+ assertEquals(HoodieSchemaType.STRING, recordKeyField.schema().getType());
+ assertFalse(recordKeyField.isNullable());
+
+ // The ordering field keeps its doc but is made nullable with a null
default, even though
+ // the table schema marks it required.
+ HoodieSchemaField orderingField = deleteLogSchema.getFields().get(1);
+ assertEquals("ts", orderingField.name());
+ assertTrue(orderingField.isNullable());
+ assertEquals(HoodieSchemaType.LONG,
orderingField.getNonNullSchema().getType());
+ assertEquals("ordering field doc", orderingField.doc().get());
+ assertEquals(HoodieSchema.NULL_VALUE, orderingField.defaultVal().get());
+ }
+
+ @Test
+ public void testCreateDeleteLogSchemaOrderingFieldVariants() {
+ HoodieSchema tableSchema = deleteLogTableSchema();
+
+ // Multiple ordering fields keep the caller's order and their own types.
+ HoodieSchema multiOrderingSchema =
HoodieSchemaUtils.createDeleteLogSchema(tableSchema, Arrays.asList("ts",
"seq"));
+ assertEquals(Arrays.asList(HoodieRecord.RECORD_KEY_METADATA_FIELD, "ts",
"seq"),
+
multiOrderingSchema.getFields().stream().map(HoodieSchemaField::name).collect(Collectors.toList()));
+ HoodieSchemaField seqField = multiOrderingSchema.getFields().get(2);
+ assertTrue(seqField.isNullable());
+ assertEquals(HoodieSchemaType.STRING,
seqField.getNonNullSchema().getType());
+ assertEquals(HoodieSchema.NULL_VALUE, seqField.defaultVal().get());
+
+ // No ordering fields leaves the record key alone.
+ assertEquals(1, HoodieSchemaUtils.createDeleteLogSchema(tableSchema,
Collections.emptyList()).getFields().size());
+
+ // An already-nullable ordering field keeps its [long, null] branch order
instead of being wrapped again.
+ HoodieSchema optionalOrderingSchema =
+ HoodieSchemaUtils.createDeleteLogSchema(tableSchema,
Collections.singletonList("opt_ts"));
+ HoodieSchemaField optTsField = optionalOrderingSchema.getFields().get(1);
+ assertEquals(Arrays.asList(HoodieSchemaType.LONG, HoodieSchemaType.NULL),
+
optTsField.schema().getTypes().stream().map(HoodieSchema::getType).collect(Collectors.toList()));
+ assertTrue(optTsField.isNullable());
+ assertEquals(HoodieSchemaType.LONG,
optTsField.getNonNullSchema().getType());
+ // Pins the current behaviour of HoodieSchemaField.of: it drops the NULL
default for a non-null-first union.
+ assertFalse(optTsField.defaultVal().isPresent());
+
+ // Logical types survive the nullable wrapping.
+ HoodieSchema logicalOrderingSchema =
HoodieSchemaUtils.createDeleteLogSchema(tableSchema, Arrays.asList("ts_ms",
"amount"));
+ HoodieSchema tsMsSchema =
logicalOrderingSchema.getFields().get(1).getNonNullSchema();
+ assertEquals(HoodieSchemaType.TIMESTAMP, tsMsSchema.getType());
+ assertEquals(HoodieSchema.TimePrecision.MILLIS, ((HoodieSchema.Timestamp)
tsMsSchema).getPrecision());
+ HoodieSchema.Decimal amountSchema = (HoodieSchema.Decimal)
logicalOrderingSchema.getFields().get(2).getNonNullSchema();
+ assertEquals(10, amountSchema.getPrecision());
+ assertEquals(2, amountSchema.getScale());
+ }
+
+ @Test
+ public void testCreateDeleteLogSchemaWithUnknownOrderingField() {
+ HoodieSchema tableSchema = deleteLogTableSchema();
+ IllegalArgumentException exception =
assertThrows(IllegalArgumentException.class,
+ () -> HoodieSchemaUtils.createDeleteLogSchema(tableSchema,
Collections.singletonList("not_a_field")));
+ assertEquals("Ordering field not_a_field not found in table schema",
exception.getMessage());
+ }
}
diff --git
a/hudi-common/src/test/java/org/apache/hudi/common/schema/TestLocalHoodieSchemaCache.java
b/hudi-common/src/test/java/org/apache/hudi/common/schema/TestLocalHoodieSchemaCache.java
new file mode 100644
index 000000000000..8acb875849f7
--- /dev/null
+++
b/hudi-common/src/test/java/org/apache/hudi/common/schema/TestLocalHoodieSchemaCache.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.common.schema;
+
+import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotSame;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class TestLocalHoodieSchemaCache {
+
+ @Test
+ public void testBasicCacheUsage() {
+ LocalHoodieSchemaCache schemaCache = LocalHoodieSchemaCache.create();
+ Integer schemaCacheNum =
schemaCache.cacheSchema(HoodieTestDataGenerator.HOODIE_SCHEMA);
+ Integer nestedSchemaCacheNum =
schemaCache.cacheSchema(HoodieTestDataGenerator.NESTED_SCHEMA);
+ Integer metaFieldsSchemaCacheNum =
schemaCache.cacheSchema(HoodieTestDataGenerator.HOODIE_SCHEMA_WITH_METADATA_FIELDS);
+ Integer decimalSchemaCacheNum =
schemaCache.cacheSchema(HoodieTestDataGenerator.HOODIE_TRIP_ENCODED_DECIMAL_SCHEMA);
+ Set<Integer> uniqueSet = new HashSet<>(
+ Arrays.asList(schemaCacheNum, nestedSchemaCacheNum,
metaFieldsSchemaCacheNum, decimalSchemaCacheNum));
+ assertEquals(4, uniqueSet.size());
+ assertTrue(schemaCache.getSchema(schemaCacheNum).isPresent());
+ assertEquals(HoodieTestDataGenerator.HOODIE_SCHEMA,
schemaCache.getSchema(schemaCacheNum).get());
+ assertTrue(schemaCache.getSchema(nestedSchemaCacheNum).isPresent());
+ assertEquals(HoodieTestDataGenerator.NESTED_SCHEMA,
schemaCache.getSchema(nestedSchemaCacheNum).get());
+ assertTrue(schemaCache.getSchema(metaFieldsSchemaCacheNum).isPresent());
+ assertEquals(HoodieTestDataGenerator.HOODIE_SCHEMA_WITH_METADATA_FIELDS,
schemaCache.getSchema(metaFieldsSchemaCacheNum).get());
+ assertTrue(schemaCache.getSchema(decimalSchemaCacheNum).isPresent());
+ assertEquals(HoodieTestDataGenerator.HOODIE_TRIP_ENCODED_DECIMAL_SCHEMA,
schemaCache.getSchema(decimalSchemaCacheNum).get());
+ assertFalse(schemaCache.getSchema(999).isPresent());
+ }
+
+ @Test
+ public void testCopiesOfSameSchema() {
+ LocalHoodieSchemaCache schemaCache = LocalHoodieSchemaCache.create();
+ HoodieSchema testSchema1 =
HoodieSchema.parse(HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA);
+ HoodieSchema testSchema2 =
HoodieSchema.parse(HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA);
+ Integer cacheNum = schemaCache.cacheSchema(testSchema1);
+ Integer secondSchemaCacheNum = schemaCache.cacheSchema(testSchema2);
+ assertEquals(cacheNum, secondSchemaCacheNum);
+ assertTrue(schemaCache.getSchema(cacheNum).isPresent());
+ assertEquals(testSchema1, schemaCache.getSchema(cacheNum).get());
+ }
+
+ @Test
+ public void testCreateReturnsIndependentCaches() {
+ LocalHoodieSchemaCache first = LocalHoodieSchemaCache.create();
+ LocalHoodieSchemaCache second = LocalHoodieSchemaCache.create();
+ assertNotSame(first, second);
+
+ Integer firstId = first.cacheSchema(HoodieTestDataGenerator.HOODIE_SCHEMA);
+ // the second cache has not seen the schema: a shared instance would
resolve the id
+ assertFalse(second.getSchema(firstId).isPresent());
+
+ // each cache starts its own id space, so the same id resolves to a
different schema per cache
+ Integer secondId =
second.cacheSchema(HoodieTestDataGenerator.NESTED_SCHEMA);
+ assertEquals(firstId, secondId);
+ assertEquals(HoodieTestDataGenerator.HOODIE_SCHEMA,
first.getSchema(firstId).get());
+ assertEquals(HoodieTestDataGenerator.NESTED_SCHEMA,
second.getSchema(secondId).get());
+ }
+}
diff --git
a/hudi-common/src/test/java/org/apache/hudi/common/table/read/lsm/TestLsmFileGroupRecordIterator.java
b/hudi-common/src/test/java/org/apache/hudi/common/table/read/lsm/TestLsmFileGroupRecordIterator.java
index 640b0b490209..8090c4859601 100644
---
a/hudi-common/src/test/java/org/apache/hudi/common/table/read/lsm/TestLsmFileGroupRecordIterator.java
+++
b/hudi-common/src/test/java/org/apache/hudi/common/table/read/lsm/TestLsmFileGroupRecordIterator.java
@@ -34,7 +34,6 @@ import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.schema.HoodieSchema;
import org.apache.hudi.common.schema.HoodieSchemaField;
import org.apache.hudi.common.schema.HoodieSchemaType;
-import org.apache.hudi.common.schema.HoodieSchemas;
import org.apache.hudi.common.schema.internal.InternalSchema;
import org.apache.hudi.common.table.HoodieTableConfig;
import org.apache.hudi.common.table.HoodieTableMetaClient;
@@ -164,20 +163,6 @@ class TestLsmFileGroupRecordIterator {
iterator.close();
}
- @Test
- void testDeleteLogSchemaUsesRecordKeyAndOrderingFields() {
- HoodieSchema deleteLogSchema =
HoodieSchemas.createDeleteLogSchema(tableSchema(), Arrays.asList("ts"));
-
- assertEquals(Arrays.asList(HoodieRecord.RECORD_KEY_METADATA_FIELD, "ts"),
deleteLogSchema.getFields().stream()
- .map(HoodieSchemaField::name)
- .collect(Collectors.toList()));
- assertEquals(HoodieSchemaType.STRING,
deleteLogSchema.getField(HoodieRecord.RECORD_KEY_METADATA_FIELD).get().schema().getType());
- HoodieSchemaField orderingField = deleteLogSchema.getField("ts").get();
- assertTrue(orderingField.schema().isNullable());
- assertEquals(HoodieSchemaType.LONG,
orderingField.getNonNullSchema().getType());
- assertEquals(HoodieSchema.NULL_VALUE, orderingField.defaultVal().get());
- }
-
@Test
void testLoserTreeMergesByRecordKeyThenMergeOrder() {
List<LsmFileGroupRecordIterator.SortedRunReader<String>> readers =
Arrays.asList(
diff --git
a/hudi-common/src/test/java/org/apache/hudi/common/table/read/lsm/TestLsmFileIterators.java
b/hudi-common/src/test/java/org/apache/hudi/common/table/read/lsm/TestLsmFileIterators.java
index dc2379b0d4cc..723a11aeb7c5 100644
---
a/hudi-common/src/test/java/org/apache/hudi/common/table/read/lsm/TestLsmFileIterators.java
+++
b/hudi-common/src/test/java/org/apache/hudi/common/table/read/lsm/TestLsmFileIterators.java
@@ -26,7 +26,7 @@ import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.schema.HoodieSchema;
import org.apache.hudi.common.schema.HoodieSchemaField;
import org.apache.hudi.common.schema.HoodieSchemaType;
-import org.apache.hudi.common.schema.HoodieSchemas;
+import org.apache.hudi.common.schema.HoodieSchemaUtils;
import org.apache.hudi.common.table.log.InstantRange;
import org.apache.hudi.common.table.read.BufferedRecord;
import org.apache.hudi.common.table.read.DeleteContext;
@@ -64,7 +64,7 @@ class TestLsmFileIterators {
RecordContext<Map<String, Object>> recordContext =
mock(RecordContext.class);
Map<String, Object> record = Collections.emptyMap();
List<String> orderingFields = Collections.singletonList("ts");
- HoodieSchema deleteLogSchema =
HoodieSchemas.createDeleteLogSchema(tableSchema(), orderingFields);
+ HoodieSchema deleteLogSchema =
HoodieSchemaUtils.createDeleteLogSchema(tableSchema(), orderingFields);
when(readerContext.getRecordContext()).thenReturn(recordContext);
when(recordContext.getValue(record, deleteLogSchema,
HoodieRecord.RECORD_KEY_METADATA_FIELD)).thenReturn("key1");
diff --git
a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/reader/HoodieFileSliceTestUtils.java
b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/reader/HoodieFileSliceTestUtils.java
index 7d875aacb7be..09b71e019dbd 100644
---
a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/reader/HoodieFileSliceTestUtils.java
+++
b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/reader/HoodieFileSliceTestUtils.java
@@ -37,7 +37,7 @@ import org.apache.hudi.common.model.HoodieLogFile;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecordLocation;
import org.apache.hudi.common.schema.HoodieSchema;
-import org.apache.hudi.common.schema.HoodieSchemas;
+import org.apache.hudi.common.schema.HoodieSchemaUtils;
import org.apache.hudi.common.table.HoodieTableConfig;
import org.apache.hudi.common.table.log.HoodieLogFormat;
import org.apache.hudi.common.table.log.HoodieLogFormatWriter;
@@ -388,7 +388,7 @@ public class HoodieFileSliceTestUtils {
HoodieSchema tableSchema,
String logInstantTime
) throws IOException {
- HoodieSchema deleteLogSchema =
HoodieSchemas.createDeleteLogSchema(tableSchema, Arrays.asList(TIMESTAMP));
+ HoodieSchema deleteLogSchema =
HoodieSchemaUtils.createDeleteLogSchema(tableSchema, Arrays.asList(TIMESTAMP));
try (HoodieAvroFileWriter writer = createNativeLogWriter(storage,
logFilePath, deleteLogSchema, logInstantTime)) {
for (IndexedRecord record : records) {
String recordKey =
record.get(record.getSchema().getField(ROW_KEY).pos()).toString();
diff --git
a/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestHoodieMetadataPayload.java
b/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestHoodieMetadataPayload.java
index 45048d4e80b0..e569d9b7f753 100644
---
a/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestHoodieMetadataPayload.java
+++
b/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestHoodieMetadataPayload.java
@@ -30,6 +30,7 @@ import
org.apache.hudi.metadata.stats.HoodieColumnRangeMetadata;
import org.apache.hudi.metadata.stats.ValueMetadata;
import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.generic.IndexedRecord;
import org.junit.jupiter.api.Test;
@@ -47,6 +48,8 @@ import static org.apache.hudi.metadata.HoodieIndexVersion.V1;
import static
org.apache.hudi.metadata.HoodieMetadataPayload.SECONDARY_INDEX_RECORD_KEY_SEPARATOR;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -368,6 +371,29 @@ public class TestHoodieMetadataPayload extends
HoodieCommonTestHarness {
((GenericRecord) projectedRecord).get("BloomFilterMetadata"));
}
+ @Test
+ public void testInsertValueFastPathOnlyForClassSchema() throws IOException {
+ HoodieMetadataPayload payload =
HoodieMetadataPayload.createBloomFilterMetadataRecord(
+ PARTITION_NAME, "file-id_1-0-1_20240101000000000.parquet",
"20240101000000000", "SIMPLE",
+ ByteBuffer.wrap("bloom-data".getBytes()), false).getData();
+
+ // The class schema singleton (and no schema) takes the reference-equality
fast path and returns the generated record.
+ IndexedRecord fastPath =
payload.getInsertValue(HoodieMetadataRecord.getClassSchema()).get();
+ assertInstanceOf(HoodieMetadataRecord.class, fastPath);
+ assertSame(HoodieMetadataRecord.getClassSchema(), fastPath.getSchema());
+ assertInstanceOf(HoodieMetadataRecord.class,
payload.getInsertValue(null).get());
+
+ // Any other instance takes the slow path, which fills a GenericRecord at
the metadata-field offsets.
+ Schema withMetaFields = HoodieSchemaUtils.addMetadataFields(
+
HoodieSchema.fromAvroSchema(HoodieMetadataRecord.getClassSchema())).toAvroSchema();
+ assertInstanceOf(GenericData.Record.class,
payload.getInsertValue(withMetaFields).get());
+
+ // So an equal but distinct copy of the bare class schema is not usable:
the fast path has to hit by identity.
+ Schema equalCopy = new
Schema.Parser().parse(HoodieMetadataRecord.getClassSchema().toString());
+ assertEquals(HoodieMetadataRecord.getClassSchema(), equalCopy);
+ assertThrows(ArrayIndexOutOfBoundsException.class, () ->
payload.getInsertValue(equalCopy));
+ }
+
@Test
public void testPayloadToStringForIndexedRecordTypes() {
HoodieMetadataPayload filesPayload =
HoodieMetadataPayload.createPartitionFilesRecord(
diff --git
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieCreateRecordUtils.scala
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieCreateRecordUtils.scala
index bd96649c19af..b8498c672559 100644
---
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieCreateRecordUtils.scala
+++
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieCreateRecordUtils.scala
@@ -19,7 +19,7 @@
package org.apache.hudi
import org.apache.hudi.DataSourceWriteOptions.INSERT_DROP_DUPS
-import org.apache.hudi.common.avro.{AvroRecordContext, AvroSchemaCache,
HoodieAvroUtils}
+import org.apache.hudi.common.avro.{AvroRecordContext, HoodieAvroUtils}
import org.apache.hudi.common.config.{RecordMergeMode, TypedProperties}
import org.apache.hudi.common.fs.FSUtils
import org.apache.hudi.common.model._
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSourceDefaults.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSourceDefaults.scala
index dadf45197414..61f52d8ff47f 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSourceDefaults.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSourceDefaults.scala
@@ -21,7 +21,7 @@ import org.apache.hudi.HoodieSparkUtils.sparkAdapter
import org.apache.hudi.common.avro.HoodieAvroUtils
import org.apache.hudi.common.config.TypedProperties
import org.apache.hudi.common.model._
-import org.apache.hudi.common.schema.{HoodieSchema, HoodieSchemaUtils =>
HoodieCommonSchemaUtils}
+import org.apache.hudi.common.schema.HoodieSchema
import org.apache.hudi.common.testutils.{OrderingFieldsTestUtils,
SchemaTestUtil}
import org.apache.hudi.common.util.Option
import
org.apache.hudi.common.util.PartitionPathEncodeUtils.DEFAULT_PARTITION_PATH
@@ -30,6 +30,7 @@ import org.apache.hudi.exception.{HoodieException,
HoodieKeyException}
import org.apache.hudi.keygen._
import org.apache.hudi.testutils.SparkDatasetTestUtils
+import org.apache.avro.Schema
import org.apache.avro.generic.GenericRecord
import org.apache.spark.sql.Row
import org.apache.spark.sql.catalyst.InternalRow
@@ -624,14 +625,14 @@ class TestDataSourceDefaults extends
ScalaAssertionSupport {
val props = new TypedProperties()
OrderingFieldsTestUtils.setOrderingFieldsConfig(props, key,
"favoriteIntNumber")
val baseOrderingVal: Object = baseRecord.get("favoriteIntNumber")
- val fieldSchema: HoodieSchema =
HoodieSchema.fromAvroSchema(baseRecord.getSchema().getField("favoriteIntNumber").schema())
+ val fieldSchema: Schema =
baseRecord.getSchema().getField("favoriteIntNumber").schema()
- val basePayload = new OverwriteWithLatestAvroPayload(baseRecord,
HoodieCommonSchemaUtils.convertValueForSpecificDataTypes(fieldSchema,
baseOrderingVal, false).asInstanceOf[Comparable[_]])
+ val basePayload = new OverwriteWithLatestAvroPayload(baseRecord,
HoodieAvroUtils.convertValueForSpecificDataTypes(fieldSchema, baseOrderingVal,
false).asInstanceOf[Comparable[_]])
val laterRecord = SchemaTestUtil
.generateAvroRecordFromJson(schema, 2, "001", "f1")
val laterOrderingVal: Object = laterRecord.get("favoriteIntNumber")
- val newerPayload = new OverwriteWithLatestAvroPayload(laterRecord,
HoodieCommonSchemaUtils.convertValueForSpecificDataTypes(fieldSchema,
laterOrderingVal, false).asInstanceOf[Comparable[_]])
+ val newerPayload = new OverwriteWithLatestAvroPayload(laterRecord,
HoodieAvroUtils.convertValueForSpecificDataTypes(fieldSchema, laterOrderingVal,
false).asInstanceOf[Comparable[_]])
// it always returns the latest payload.
val preCombinedPayload = basePayload.preCombine(newerPayload)
@@ -640,7 +641,7 @@ class TestDataSourceDefaults extends ScalaAssertionSupport {
}
@Test def testDefaultHoodieRecordPayloadCombineAndGetUpdateValue(): Unit = {
- val fieldSchema: HoodieSchema =
HoodieSchema.fromAvroSchema(baseRecord.getSchema().getField("favoriteIntNumber").schema())
+ val fieldSchema: Schema =
baseRecord.getSchema().getField("favoriteIntNumber").schema()
val props = HoodiePayloadConfig.newBuilder()
.withPayloadOrderingFields("favoriteIntNumber").build().getProps;
@@ -653,10 +654,10 @@ class TestDataSourceDefaults extends
ScalaAssertionSupport {
val earlierOrderingVal: Object = earlierRecord.get("favoriteIntNumber")
val laterPayload = new DefaultHoodieRecordPayload(laterRecord,
- HoodieCommonSchemaUtils.convertValueForSpecificDataTypes(fieldSchema,
laterOrderingVal, false).asInstanceOf[Comparable[_]])
+ HoodieAvroUtils.convertValueForSpecificDataTypes(fieldSchema,
laterOrderingVal, false).asInstanceOf[Comparable[_]])
val earlierPayload = new DefaultHoodieRecordPayload(earlierRecord,
- HoodieCommonSchemaUtils.convertValueForSpecificDataTypes(fieldSchema,
earlierOrderingVal, false).asInstanceOf[Comparable[_]])
+ HoodieAvroUtils.convertValueForSpecificDataTypes(fieldSchema,
earlierOrderingVal, false).asInstanceOf[Comparable[_]])
// it will provide the record with greatest combine value
val preCombinedPayload = laterPayload.preCombine(earlierPayload)
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/HoodieSparkSqlTestBase.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/HoodieSparkSqlTestBase.scala
index b5f5662c5258..ed775afc13be 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/HoodieSparkSqlTestBase.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/HoodieSparkSqlTestBase.scala
@@ -19,7 +19,6 @@ package org.apache.spark.sql.hudi.common
import org.apache.hudi.{DefaultSparkRecordMerger, HoodieSparkUtils}
import org.apache.hudi.HoodieFileIndex.DataSkippingFailureMode
-import org.apache.hudi.common.avro.AvroSchemaCache
import org.apache.hudi.common.config.{HoodieCommonConfig,
HoodieMetadataConfig, HoodieStorageConfig}
import org.apache.hudi.common.engine.HoodieLocalEngineContext
import org.apache.hudi.common.model.{FileSlice, HoodieAvroRecordMerger,
HoodieLogFile, HoodieRecord, HoodieReplaceCommitMetadata}
diff --git
a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestJsonKafkaSource.java
b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestJsonKafkaSource.java
index 71926f32cd48..1e0bbac0e759 100644
---
a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestJsonKafkaSource.java
+++
b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestJsonKafkaSource.java
@@ -21,6 +21,7 @@ package org.apache.hudi.utilities.sources;
import org.apache.hudi.HoodieSchemaUtils;
import org.apache.hudi.client.WriteStatus;
import org.apache.hudi.client.common.HoodieSparkEngineContext;
+import org.apache.hudi.common.avro.HoodieAvroUtils;
import org.apache.hudi.common.config.TypedProperties;
import org.apache.hudi.common.model.HoodieAvroIndexedRecord;
import org.apache.hudi.common.model.HoodieKey;
@@ -296,7 +297,7 @@ public class TestJsonKafkaSource extends
BaseTestKafkaSource {
double maxVal = Math.pow(10, decSchema.getPrecision() -
decSchema.getScale());
double minVal = maxVal * 0.1;
for (GenericRecord record : records) {
- BigDecimal dec =
org.apache.hudi.common.schema.HoodieSchemaUtils.convertBytesToBigDecimal(((ByteBuffer)
record.get(fieldname)).array(), decSchema);
+ BigDecimal dec = HoodieAvroUtils.convertBytesToBigDecimal(((ByteBuffer)
record.get(fieldname)).array(), decSchema);
double doubleValue = dec.doubleValue();
assertTrue(doubleValue <= maxVal && doubleValue >= minVal);
}
