This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new c32b64e98a3a fix(common): normalize Avro ordering values for record
merging (#20544)
c32b64e98a3a is described below
commit c32b64e98a3a3b637f793a77c83f849bf3655c57
Author: Shuo Cheng <[email protected]>
AuthorDate: Thu Nov 12 11:56:30 2026 +0000
fix(common): normalize Avro ordering values for record merging (#20544)
* fix(common): normalize Avro ordering values for record merging
* fix(common): normalize Avro delete ordering values using field schemas
* update
* fix(ci): preserve locally built Hudi artifacts when switching JDKs
* fix comment
* fix comment
* refactor(common): return normalized delete timestamps directly
* refactor(common): prebuild delete ordering field converters
---
.github/workflows/hudi_trino_ci.yml | 2 +-
.github/workflows/hudi_trino_e2e.yml | 2 +-
.../common/avro/AvroOrderingValueConverter.java | 136 ++++++++++++
.../apache/hudi/common/avro/AvroRecordContext.java | 52 ++++-
.../hudi/common/avro/HoodieAvroReaderContext.java | 11 +-
.../apache/hudi/common/avro/HoodieAvroUtils.java | 10 +-
.../hudi/common/model/HoodieAvroIndexedRecord.java | 2 +-
.../table/read/BufferedRecordMergerFactory.java | 2 +-
.../hudi/common/avro/TestAvroRecordContext.java | 233 ++++++++++++++++++++-
.../common/avro/TestHoodieAvroReaderContext.java | 25 +++
.../TestBufferedRecordSerializer.java | 5 +-
.../read/TestBufferedRecordMergerFactory.java | 25 +++
.../buffer/TestFileGroupRecordBufferLoader.java | 1 +
.../buffer/TestKeyBasedFileGroupRecordBuffer.java | 83 ++++++++
.../TestSortedKeyBasedFileGroupRecordBuffer.java | 1 +
...TestStreamingKeyBasedFileGroupRecordBuffer.java | 4 +
.../TestHoodieBackedTableMetadataDataCleanup.java | 1 +
.../hudi/functional/TestBufferedRecordMerger.java | 1 +
.../others/TestMergeModeEventTimeOrdering.scala | 31 +++
.../sql/hudi/dml/others/TestUpdateTable.scala | 47 +++++
.../trino/plugin/hudi/HudiPageSourceProvider.java | 3 +-
.../plugin/hudi/reader/HudiTrinoReaderContext.java | 17 +-
.../hudi/reader/TestHudiTrinoReaderContext.java | 146 +++++++++++++
23 files changed, 821 insertions(+), 19 deletions(-)
diff --git a/.github/workflows/hudi_trino_ci.yml
b/.github/workflows/hudi_trino_ci.yml
index 8d60f8c30810..35cb5b5f26af 100644
--- a/.github/workflows/hudi_trino_ci.yml
+++ b/.github/workflows/hudi_trino_ci.yml
@@ -93,11 +93,11 @@ jobs:
# they live behind the hudi-trino-tests profile but the test step
needs them.
run: mvn $MVN_ARGS install -pl
:hudi-common,:hudi-hive-sync,:hudi-io,:hudi-sync-common,:hudi-client-common,:hudi-java-client
-am -Dmaven.test.skip=true -Drat.skip -Dcheckstyle.skip
- name: Set up JDK 25
+ # Restoring the Maven cache again would overwrite the Hudi artifacts
installed above.
uses: actions/setup-java@v5
with:
java-version: '25'
distribution: 'temurin'
- cache: maven
- name: Purge Trino artifacts from the local m2
# Artifacts an older pin left behind carry the same SNAPSHOT
coordinates as the current ones.
run: rm -rf ~/.m2/repository/io/trino
diff --git a/.github/workflows/hudi_trino_e2e.yml
b/.github/workflows/hudi_trino_e2e.yml
index e1a5d94c594a..9ec8fd393d09 100644
--- a/.github/workflows/hudi_trino_e2e.yml
+++ b/.github/workflows/hudi_trino_e2e.yml
@@ -114,11 +114,11 @@ jobs:
run:
mvn clean install -T 2 $SCALA_PROFILE -Dspark4.0 -Dflink1.20
-Pintegration-tests -DskipTests=true -Ddocker.compose.skip=true $MVN_ARGS
- name: Set up JDK 25
+ # Restoring the Maven cache again would overwrite the Hudi artifacts
installed above.
uses: actions/setup-java@v5
with:
java-version: '25'
distribution: 'temurin'
- cache: maven
- name: Purge Trino artifacts from the local m2
# Artifacts an older pin left behind carry the same SNAPSHOT
coordinates as the current ones.
run: rm -rf ~/.m2/repository/io/trino
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/avro/AvroOrderingValueConverter.java
b/hudi-common/src/main/java/org/apache/hudi/common/avro/AvroOrderingValueConverter.java
new file mode 100644
index 000000000000..89c7a03c274f
--- /dev/null
+++
b/hudi-common/src/main/java/org/apache/hudi/common/avro/AvroOrderingValueConverter.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.common.avro;
+
+import org.apache.hudi.common.schema.HoodieSchema;
+import org.apache.hudi.common.schema.HoodieSchemaField;
+import org.apache.hudi.common.schema.HoodieSchemaUtils;
+import org.apache.hudi.common.util.collection.ArrayComparable;
+import org.apache.hudi.common.util.collection.Pair;
+
+import org.apache.avro.Conversions.DecimalConversion;
+import org.apache.avro.LogicalType;
+import org.apache.avro.LogicalTypes;
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericFixed;
+import org.apache.avro.util.Utf8;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.sql.Date;
+import java.sql.Timestamp;
+import java.time.LocalDate;
+import java.util.List;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import static org.apache.hudi.common.avro.HoodieAvroUtils.extractEpochDay;
+import static org.apache.hudi.common.avro.HoodieAvroUtils.extractEpochMicros;
+import static org.apache.hudi.common.avro.HoodieAvroUtils.extractEpochMillis;
+import static
org.apache.hudi.common.avro.HoodieAvroUtils.extractLocalEpochMicros;
+import static
org.apache.hudi.common.avro.HoodieAvroUtils.extractLocalEpochMillis;
+
+/**
+ * Prebuilds a converter for each delete ordering field to match the
comparison representation of
+ * {@link HoodieAvroUtils#convertValueForAvroLogicalTypes(Schema, Object,
boolean)}.
+ * Preserves null field values and converts strings to Avro's comparable
representation.
+ */
+public class AvroOrderingValueConverter {
+ private final List<Function<Comparable, Comparable>> converters;
+
+ private AvroOrderingValueConverter(HoodieSchema dataSchema, List<String>
orderingFieldNames, boolean consistentLogicalTimestampEnabled) {
+ this.converters = orderingFieldNames.stream()
+ .map(field -> HoodieSchemaUtils.getNestedField(dataSchema, field)
+ .map(Pair::getRight).map(HoodieSchemaField::getNonNullSchema)
+ .orElseThrow(() -> new IllegalArgumentException("Ordering field '"
+ field + "' not found in table schema")))
+ .map(schema -> createConverter(schema.toAvroSchema(),
consistentLogicalTimestampEnabled))
+ .map(converter -> (Function<Comparable, Comparable>) value -> value ==
null ? null : converter.apply(value))
+ .collect(Collectors.toList());
+ }
+
+ public static AvroOrderingValueConverter create(HoodieSchema dataSchema,
List<String> orderingFieldNames, boolean consistentLogicalTimestampEnabled) {
+ return new AvroOrderingValueConverter(dataSchema, orderingFieldNames,
consistentLogicalTimestampEnabled);
+ }
+
+ public Comparable convert(Comparable value) {
+ if (converters.isEmpty()) {
+ return value;
+ }
+ return value instanceof ArrayComparable
+ ? ((ArrayComparable) value).apply(converters)
+ : converters.get(0).apply(value);
+ }
+
+ private static Function<Comparable, Comparable> createConverter(Schema
schema, boolean consistentLogicalTimestampEnabled) {
+ LogicalType logicalType = schema.getLogicalType();
+ if (logicalType == LogicalTypes.date()) {
+ // Spark/Flink supply epoch days; Avro supplies LocalDate. Callers may
supply java.sql.Date.
+ return value -> value instanceof Date ? ((Date) value).toLocalDate() :
LocalDate.ofEpochDay(extractEpochDay(value));
+ }
+ if (logicalType == LogicalTypes.timestampMillis()) {
+ return consistentLogicalTimestampEnabled
+ ? value -> new Timestamp(toEpochMillis(value)) :
AvroOrderingValueConverter::toEpochMillis;
+ }
+ if (logicalType == LogicalTypes.timestampMicros()) {
+ // Match the shared normalizer's millisecond truncation, including
negative timestamps.
+ return consistentLogicalTimestampEnabled
+ ? value -> new Timestamp(toEpochMicros(value) / 1000) :
AvroOrderingValueConverter::toEpochMicros;
+ }
+ // A caller-supplied Timestamp represents wall-clock fields for a local
timestamp schema.
+ if (logicalType == LogicalTypes.localTimestampMillis()) {
+ return value -> extractLocalEpochMillis(value instanceof Timestamp ?
((Timestamp) value).toLocalDateTime() : value);
+ }
+ if (logicalType == LogicalTypes.localTimestampMicros()) {
+ return value -> extractLocalEpochMicros(value instanceof Timestamp ?
((Timestamp) value).toLocalDateTime() : value);
+ }
+ if (logicalType instanceof LogicalTypes.Decimal) {
+ // Spark/Flink and Avro extraction produce BigDecimal, preserved by
DecimalWrapper decoding.
+ DecimalConversion decimalConversion = new DecimalConversion();
+ if (schema.getType() == Schema.Type.FIXED) {
+ return value -> value instanceof BigDecimal ? value
+ : decimalConversion.fromFixed((GenericFixed) value, schema,
logicalType);
+ } else if (schema.getType() == Schema.Type.BYTES) {
+ return value -> {
+ if (value instanceof BigDecimal) {
+ return value;
+ }
+ ByteBuffer buffer = (ByteBuffer) value;
+ BigDecimal convertedValue = decimalConversion.fromBytes(buffer,
schema, logicalType);
+ buffer.rewind();
+ return convertedValue;
+ };
+ }
+ }
+ if (schema.getType() == Schema.Type.STRING) {
+ // Spark/Flink Java value extraction and StringWrapper decoding produce
String.
+ return value -> value instanceof String ? new Utf8((String) value) :
value;
+ }
+ return value -> value;
+ }
+
+ private static long toEpochMillis(Comparable value) {
+ // Avro extraction with consistent logical timestamps enabled and wrapper
decoding yield Timestamp.
+ return value instanceof Timestamp ? ((Timestamp) value).getTime() :
extractEpochMillis(value);
+ }
+
+ private static long toEpochMicros(Comparable value) {
+ return extractEpochMicros(value instanceof Timestamp ? ((Timestamp)
value).toInstant() : value);
+ }
+}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/avro/AvroRecordContext.java
b/hudi-common/src/main/java/org/apache/hudi/common/avro/AvroRecordContext.java
index 8f4a5fa2ddf8..d07dea1ce1a9 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/avro/AvroRecordContext.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/avro/AvroRecordContext.java
@@ -19,6 +19,7 @@
package org.apache.hudi.common.avro;
+import org.apache.hudi.common.config.TypedProperties;
import org.apache.hudi.common.engine.RecordContext;
import org.apache.hudi.common.model.HoodieAvroIndexedRecord;
import org.apache.hudi.common.model.HoodieEmptyRecord;
@@ -30,8 +31,10 @@ import org.apache.hudi.common.schema.HoodieSchemaField;
import org.apache.hudi.common.table.HoodieTableConfig;
import org.apache.hudi.common.table.read.BufferedRecord;
import org.apache.hudi.common.util.AvroJavaTypeConverter;
+import org.apache.hudi.common.util.ConfigUtils;
import org.apache.hudi.common.util.HoodieRecordUtils;
import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.OrderingValues;
import org.apache.hudi.exception.HoodieException;
import org.apache.avro.generic.GenericData;
@@ -40,6 +43,7 @@ import org.apache.avro.generic.IndexedRecord;
import org.apache.avro.util.Utf8;
import java.io.IOException;
+import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.function.UnaryOperator;
@@ -57,15 +61,19 @@ public class AvroRecordContext extends
RecordContext<IndexedRecord> {
}
private final String payloadClass;
+ private final boolean consistentLogicalTimestampEnabled;
+ private AvroOrderingValueConverter orderingValueConverter;
- public AvroRecordContext(HoodieTableConfig tableConfig, String payloadClass)
{
+ public AvroRecordContext(HoodieTableConfig tableConfig, String payloadClass,
TypedProperties props) {
super(tableConfig, new AvroJavaTypeConverter());
this.payloadClass = payloadClass;
+ this.consistentLogicalTimestampEnabled =
ConfigUtils.shouldKeepConsistentLogicalTimestamp(props);
}
public AvroRecordContext() {
super(new AvroJavaTypeConverter());
this.payloadClass = null;
+ this.consistentLogicalTimestampEnabled = false;
}
public static Object getFieldValueFromIndexedRecord(
@@ -107,6 +115,36 @@ public class AvroRecordContext extends
RecordContext<IndexedRecord> {
return getFieldValueFromIndexedRecord(record, fieldName);
}
+ @Override
+ public Comparable getOrderingValue(IndexedRecord record, HoodieSchema
schema, List<String> orderingFieldNames) {
+ return OrderingValues.create(orderingFieldNames, field -> {
+ Object value = getOrderingValueFromIndexedRecord(record, field,
consistentLogicalTimestampEnabled);
+ return value == null ? OrderingValues.getDefault() :
ensureComparability(value);
+ });
+ }
+
+ @Override
+ public Comparable getOrderingValue(IndexedRecord record, HoodieSchema
schema, String[] orderingFieldNames) {
+ return orderingFieldNames == null
+ ? OrderingValues.getDefault()
+ : OrderingValues.create(orderingFieldNames, field -> {
+ Object value = getOrderingValueFromIndexedRecord(record, field,
consistentLogicalTimestampEnabled);
+ return value == null ? OrderingValues.getDefault() :
ensureComparability(value);
+ });
+ }
+
+ /**
+ * Extracts an ordering field from a generic Avro record, normalizing its
logical type.
+ *
+ * @param record the record to read; must be a {@link GenericRecord}
+ * @param fieldName the field name, which may be a nested path
+ * @param consistentLogicalTimestampEnabled whether to use consistent
logical timestamp conversion
+ * @return the ordering value, or null if the field or its value is missing
+ */
+ public static Object getOrderingValueFromIndexedRecord(IndexedRecord record,
String fieldName, boolean consistentLogicalTimestampEnabled) {
+ return HoodieAvroUtils.getNestedFieldVal((GenericRecord) record,
fieldName, true, consistentLogicalTimestampEnabled);
+ }
+
@Override
public String getMetaFieldValue(IndexedRecord record, int pos) {
return record.get(pos).toString();
@@ -203,6 +241,18 @@ public class AvroRecordContext extends
RecordContext<IndexedRecord> {
return record -> HoodieAvroUtils.rewriteRecordWithNewSchema(record,
to.toAvroSchema(), renamedColumns);
}
+ /**
+ * Initializes schema-aware conversion so delete values use the same logical
types as data records.
+ */
+ public void initOrderingValueConverter(HoodieSchema dataSchema, List<String>
orderingFieldNames) {
+ orderingValueConverter = AvroOrderingValueConverter.create(dataSchema,
orderingFieldNames, consistentLogicalTimestampEnabled);
+ }
+
+ @Override
+ public Comparable convertOrderingValueToEngineType(Comparable value) {
+ return orderingValueConverter.convert(value);
+ }
+
@Override
public Comparable convertValueToEngineType(Comparable value) {
if (value instanceof String) {
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/avro/HoodieAvroReaderContext.java
b/hudi-common/src/main/java/org/apache/hudi/common/avro/HoodieAvroReaderContext.java
index 788810b7ba3f..be60facac9b7 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/avro/HoodieAvroReaderContext.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/avro/HoodieAvroReaderContext.java
@@ -39,6 +39,7 @@ import org.apache.hudi.common.table.HoodieTableConfig;
import org.apache.hudi.common.table.log.InstantRange;
import org.apache.hudi.common.table.read.BufferedRecord;
import org.apache.hudi.common.table.read.BufferedRecordSerializer;
+import org.apache.hudi.common.table.read.FileGroupReaderSchemaHandler;
import org.apache.hudi.common.util.ConfigUtils;
import org.apache.hudi.common.util.HoodieRecordUtils;
import org.apache.hudi.common.util.Option;
@@ -153,11 +154,19 @@ public class HoodieAvroReaderContext extends
HoodieReaderContext<IndexedRecord>
Map<StoragePath, HoodieAvroFileReader> reusableFileReaders,
String payloadClassName,
HoodieConfig hoodieReaderConfig) {
- super(storageConfiguration, tableConfig, instantRangeOpt, filterOpt, new
AvroRecordContext(tableConfig, payloadClassName), hoodieReaderConfig);
+ super(storageConfiguration, tableConfig, instantRangeOpt, filterOpt,
+ new AvroRecordContext(tableConfig, payloadClassName,
ConfigUtils.getMergeProps(hoodieReaderConfig.getProps(), tableConfig)),
hoodieReaderConfig);
this.reusableFileReaders = reusableFileReaders;
this.isMultiFormat = tableConfig.isMultipleBaseFileFormatsEnabled();
}
+ @Override
+ public void setSchemaHandler(FileGroupReaderSchemaHandler<IndexedRecord>
schemaHandler) {
+ super.setSchemaHandler(schemaHandler);
+ List<String> orderingFieldNames =
HoodieRecordUtils.getOrderingFieldNames(getMergeMode(), tableConfig);
+ ((AvroRecordContext)
recordContext).initOrderingValueConverter(schemaHandler.getTableSchema(),
orderingFieldNames);
+ }
+
@Override
public ClosableIterator<IndexedRecord> getFileRecordIterator(
StoragePathInfo storagePathInfo, long start, long length, HoodieSchema
dataSchema, HoodieSchema requiredSchema,
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/avro/HoodieAvroUtils.java
b/hudi-common/src/main/java/org/apache/hudi/common/avro/HoodieAvroUtils.java
index 21485fe7876c..5f768ab203d2 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/avro/HoodieAvroUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/avro/HoodieAvroUtils.java
@@ -906,7 +906,7 @@ public class HoodieAvroUtils {
// fastReaderEnabled=true), and return the canonical primitive for the same
underlying bytes.
// See the javadoc on convertValueForAvroLogicalTypes for context.
- private static long extractEpochDay(Object fieldValue) {
+ static long extractEpochDay(Object fieldValue) {
if (fieldValue instanceof LocalDate) {
return ((LocalDate) fieldValue).toEpochDay();
}
@@ -916,7 +916,7 @@ public class HoodieAvroUtils {
return Long.parseLong(fieldValue.toString());
}
- private static long extractEpochMillis(Object fieldValue) {
+ static long extractEpochMillis(Object fieldValue) {
if (fieldValue instanceof Instant) {
return ((Instant) fieldValue).toEpochMilli();
}
@@ -926,7 +926,7 @@ public class HoodieAvroUtils {
return Long.parseLong(fieldValue.toString());
}
- private static long extractEpochMicros(Object fieldValue) {
+ static long extractEpochMicros(Object fieldValue) {
if (fieldValue instanceof Instant) {
Instant instant = (Instant) fieldValue;
return Math.addExact(Math.multiplyExact(instant.getEpochSecond(),
1_000_000L), instant.getNano() / 1000L);
@@ -937,7 +937,7 @@ public class HoodieAvroUtils {
return Long.parseLong(fieldValue.toString());
}
- private static long extractLocalEpochMillis(Object fieldValue) {
+ static long extractLocalEpochMillis(Object fieldValue) {
if (fieldValue instanceof LocalDateTime) {
return ((LocalDateTime)
fieldValue).toInstant(ZoneOffset.UTC).toEpochMilli();
}
@@ -947,7 +947,7 @@ public class HoodieAvroUtils {
return Long.parseLong(fieldValue.toString());
}
- private static long extractLocalEpochMicros(Object fieldValue) {
+ static long extractLocalEpochMicros(Object fieldValue) {
if (fieldValue instanceof LocalDateTime) {
Instant instant = ((LocalDateTime) fieldValue).toInstant(ZoneOffset.UTC);
return Math.addExact(Math.multiplyExact(instant.getEpochSecond(),
1_000_000L), instant.getNano() / 1000L);
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieAvroIndexedRecord.java
b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieAvroIndexedRecord.java
index eda401c5b8e5..c17f4b824ff2 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieAvroIndexedRecord.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieAvroIndexedRecord.java
@@ -315,7 +315,7 @@ public class HoodieAvroIndexedRecord extends
HoodieRecord<IndexedRecord> {
KeyGeneratorOptions.KEYGENERATOR_CONSISTENT_LOGICAL_TIMESTAMP_ENABLED.defaultValue()));
return OrderingValues.create(
orderingFields,
- field -> (Comparable<?>)
HoodieAvroUtils.getNestedFieldVal((GenericRecord) data, field, true,
consistentLogicalTimestampEnabled));
+ field -> (Comparable<?>)
AvroRecordContext.getOrderingValueFromIndexedRecord(data, field,
consistentLogicalTimestampEnabled));
}
private void decodeRecord(Schema recordSchema) {
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/read/BufferedRecordMergerFactory.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/read/BufferedRecordMergerFactory.java
index 4484abaefee9..45b0d2801086 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/read/BufferedRecordMergerFactory.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/read/BufferedRecordMergerFactory.java
@@ -493,7 +493,7 @@ public class BufferedRecordMergerFactory {
return Option.empty();
}
Comparable existingOrderingVal = existingRecord.getOrderingValue();
- Comparable deleteOrderingVal =
recordContext.convertOrderingValueToEngineType(deleteRecord.getOrderingValue());
+ Comparable deleteOrderingVal =
recordContext.getOrderingValue(deleteRecord);
// Checks the ordering value does not equal to 0
// because we use 0 as the default value which means natural order
boolean chooseExisting = !OrderingValues.isDefault(deleteOrderingVal)
diff --git
a/hudi-common/src/test/java/org/apache/hudi/common/avro/TestAvroRecordContext.java
b/hudi-common/src/test/java/org/apache/hudi/common/avro/TestAvroRecordContext.java
index 6972b93fc735..b35fdb74357e 100644
---
a/hudi-common/src/test/java/org/apache/hudi/common/avro/TestAvroRecordContext.java
+++
b/hudi-common/src/test/java/org/apache/hudi/common/avro/TestAvroRecordContext.java
@@ -19,10 +19,21 @@
package org.apache.hudi.common.avro;
+import org.apache.hudi.common.config.TypedProperties;
+import org.apache.hudi.common.model.DeleteRecord;
+import org.apache.hudi.common.model.HoodieAvroIndexedRecord;
import org.apache.hudi.common.model.MetaFieldsMode;
import org.apache.hudi.common.schema.HoodieSchema;
import org.apache.hudi.common.table.HoodieTableConfig;
+import org.apache.hudi.common.table.log.block.HoodieDeleteBlock;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.OrderingValues;
+import org.apache.hudi.common.util.collection.ArrayComparable;
+import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.keygen.constant.KeyGeneratorOptions;
+import org.apache.avro.Conversions;
+import org.apache.avro.LogicalTypes;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericRecord;
@@ -31,9 +42,22 @@ import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
+import org.junit.jupiter.params.provider.ValueSource;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.sql.Date;
+import java.sql.Timestamp;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
import java.util.Map;
import java.util.stream.Stream;
@@ -61,6 +85,213 @@ class TestAvroRecordContext {
+
"{\"name\":\"rec_map\",\"type\":[\"null\",{\"type\":\"map\",\"values\":{\"type\":\"record\","
+
"\"name\":\"inner\",\"fields\":[{\"name\":\"x\",\"type\":\"int\"}]}}],\"default\":null}]}");
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void testOrderingValuesMatchIncomingRecord(boolean consistentTimestamp) {
+ Schema schema = new
Schema.Parser().parse("{\"type\":\"record\",\"name\":\"ordering\",\"fields\":["
+ +
"{\"name\":\"ts\",\"type\":{\"type\":\"long\",\"logicalType\":\"timestamp-micros\"}},"
+ +
"{\"name\":\"date\",\"type\":{\"type\":\"int\",\"logicalType\":\"date\"}},"
+ +
"{\"name\":\"decimal\",\"type\":{\"type\":\"fixed\",\"name\":\"fixed_decimal\",\"size\":5,"
+ + "\"logicalType\":\"decimal\",\"precision\":10,\"scale\":2}},"
+ +
"{\"name\":\"nested\",\"type\":{\"type\":\"record\",\"name\":\"inner_date\",\"fields\":["
+ +
"{\"name\":\"date\",\"type\":{\"type\":\"int\",\"logicalType\":\"date\"}}]}}]}");
+ GenericRecord record = new GenericData.Record(schema);
+ record.put("ts", 1790812800000000L);
+ record.put("date", 20727);
+ Schema decimalSchema = schema.getField("decimal").schema();
+ record.put("decimal", new Conversions.DecimalConversion().toFixed(new
BigDecimal("1.50"), decimalSchema, LogicalTypes.decimal(10, 2)));
+ GenericRecord nested = new
GenericData.Record(schema.getField("nested").schema());
+ nested.put("date", 20727);
+ record.put("nested", nested);
+ HoodieSchema hoodieSchema = HoodieSchema.fromAvroSchema(schema);
+ TypedProperties props = new TypedProperties();
+
props.setProperty(KeyGeneratorOptions.KEYGENERATOR_CONSISTENT_LOGICAL_TIMESTAMP_ENABLED.key(),
String.valueOf(consistentTimestamp));
+ AvroRecordContext context = new AvroRecordContext(new HoodieTableConfig(),
null, props);
+
+ for (String field : Arrays.asList("ts", "date", "decimal", "nested.date"))
{
+ Comparable incoming = new
HoodieAvroIndexedRecord(record).getOrderingValue(hoodieSchema, props, new
String[] {field});
+ assertEquals(incoming, context.getOrderingValue(record, hoodieSchema,
new String[] {field}), field);
+ assertEquals(incoming, context.getOrderingValue(record, hoodieSchema,
Arrays.asList(field)), field);
+ }
+ assertEquals(OrderingValues.getDefault(), context.getOrderingValue(record,
hoodieSchema, new String[] {"missing"}));
+ assertNull(new
HoodieAvroIndexedRecord(record).getOrderingValue(hoodieSchema, props, new
String[] {"missing"}));
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void testDeleteOrderingValuesMatchData(boolean consistentTimestamp) throws
IOException {
+ Schema schema = new
Schema.Parser().parse("{\"type\":\"record\",\"name\":\"delete_ordering\",\"fields\":["
+ +
"{\"name\":\"date\",\"type\":[\"null\",{\"type\":\"int\",\"logicalType\":\"date\"}],\"default\":null},"
+ +
"{\"name\":\"ts_millis\",\"type\":{\"type\":\"long\",\"logicalType\":\"timestamp-millis\"}},"
+ +
"{\"name\":\"ts_micros\",\"type\":{\"type\":\"long\",\"logicalType\":\"timestamp-micros\"}},"
+ +
"{\"name\":\"decimal_fixed\",\"type\":{\"type\":\"fixed\",\"name\":\"decimal\",\"size\":5,"
+ + "\"logicalType\":\"decimal\",\"precision\":10,\"scale\":2}},"
+ +
"{\"name\":\"decimal_bytes\",\"type\":{\"type\":\"bytes\",\"logicalType\":\"decimal\",\"precision\":10,\"scale\":2}},"
+ +
"{\"name\":\"plain_int\",\"type\":\"int\"},{\"name\":\"plain_long\",\"type\":\"long\"},"
+ + "{\"name\":\"text\",\"type\":\"string\"},"
+ +
"{\"name\":\"nested\",\"type\":{\"type\":\"record\",\"name\":\"nested_ordering\",\"fields\":["
+ +
"{\"name\":\"date\",\"type\":{\"type\":\"int\",\"logicalType\":\"date\"}}]}}]}");
+ Map<String, Comparable> deleteValues = new LinkedHashMap<>();
+ deleteValues.put("date", 20727);
+ deleteValues.put("ts_millis", 1790812800123L);
+ deleteValues.put("ts_micros", 1790812800123456L);
+ deleteValues.put("decimal_fixed", new BigDecimal("1.50"));
+ deleteValues.put("decimal_bytes", new BigDecimal("2.50"));
+ deleteValues.put("plain_int", 20727);
+ deleteValues.put("plain_long", 1790812800123456L);
+ deleteValues.put("text", "value");
+ GenericRecord record = new GenericData.Record(schema);
+ deleteValues.forEach(record::put);
+ record.put("text", new Utf8("value"));
+ Schema fixedSchema = schema.getField("decimal_fixed").schema();
+ Schema bytesSchema = schema.getField("decimal_bytes").schema();
+ record.put("decimal_fixed", new Conversions.DecimalConversion().toFixed(
+ (BigDecimal) deleteValues.get("decimal_fixed"), fixedSchema,
fixedSchema.getLogicalType()));
+ record.put("decimal_bytes", new Conversions.DecimalConversion().toBytes(
+ (BigDecimal) deleteValues.get("decimal_bytes"), bytesSchema,
bytesSchema.getLogicalType()));
+ GenericRecord nested = new
GenericData.Record(schema.getField("nested").schema());
+ nested.put("date", 20727);
+ record.put("nested", nested);
+ deleteValues.put("nested.date", 20727);
+ HoodieSchema hoodieSchema = HoodieSchema.fromAvroSchema(schema);
+ TypedProperties props = new TypedProperties();
+
props.setProperty(KeyGeneratorOptions.KEYGENERATOR_CONSISTENT_LOGICAL_TIMESTAMP_ENABLED.key(),
String.valueOf(consistentTimestamp));
+ AvroRecordContext context = new AvroRecordContext(new HoodieTableConfig(),
null, props);
+
+ for (Map.Entry<String, Comparable> entry : deleteValues.entrySet()) {
+ List<String> fields = Collections.singletonList(entry.getKey());
+ context.initOrderingValueConverter(hoodieSchema, fields);
+ Comparable dataOrdering = context.getOrderingValue(record, hoodieSchema,
fields);
+ Comparable deleteOrdering =
context.getOrderingValue(roundTripDelete(entry.getValue()));
+ assertEquals(0, dataOrdering.compareTo(deleteOrdering), entry.getKey());
+ assertNull(context.convertOrderingValueToEngineType(null),
entry.getKey());
+ if (entry.getValue() instanceof BigDecimal) {
+ assertEquals(dataOrdering,
context.convertOrderingValueToEngineType((Comparable)
record.get(entry.getKey())));
+ }
+ }
+
+ // SQL date/timestamp wrappers decode to different Java representations
than primitive wrappers.
+ context.initOrderingValueConverter(hoodieSchema,
Collections.singletonList("date"));
+ LocalDate date = LocalDate.ofEpochDay(20727);
+ assertEquals(date,
context.getOrderingValue(roundTripDelete(Date.valueOf(date))));
+ assertEquals(date, context.convertOrderingValueToEngineType(date));
+ assertEquals(date, context.convertOrderingValueToEngineType("20727"));
+ assertEquals(date, context.convertOrderingValueToEngineType(new
Utf8("20727")));
+ assertThrows(NumberFormatException.class, () ->
context.convertOrderingValueToEngineType("invalid"));
+ for (String field : Arrays.asList("ts_millis", "ts_micros")) {
+ List<String> fields = Collections.singletonList(field);
+ context.initOrderingValueConverter(hoodieSchema, fields);
+ long rawValue = (long) deleteValues.get(field);
+ Instant instant = field.equals("ts_millis") ?
Instant.ofEpochMilli(rawValue)
+ : Instant.ofEpochSecond(rawValue / 1_000_000, (rawValue % 1_000_000)
* 1_000);
+ Comparable dataOrdering = context.getOrderingValue(record, hoodieSchema,
fields);
+ assertEquals(0,
dataOrdering.compareTo(context.getOrderingValue(roundTripDelete(Timestamp.from(instant)))));
+ }
+
+ List<String> fields = new ArrayList<>(deleteValues.keySet());
+ context.initOrderingValueConverter(hoodieSchema, fields);
+ Comparable multipleValues =
OrderingValues.create(deleteValues.values().toArray(new Comparable[0]));
+ Comparable decodedValues =
roundTripDelete(multipleValues).getOrderingValue();
+ Comparable dataOrdering = context.getOrderingValue(record, hoodieSchema,
fields);
+ assertEquals(0,
dataOrdering.compareTo(context.convertOrderingValueToEngineType(decodedValues)));
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"local-timestamp-millis", "local-timestamp-micros"})
+ void testLocalTimestampDeleteOrdering(String logicalType) throws IOException
{
+ Schema schema = new
Schema.Parser().parse("{\"type\":\"record\",\"name\":\"local_timestamp\",\"fields\":["
+ + "{\"name\":\"ts\",\"type\":{\"type\":\"long\",\"logicalType\":\"" +
logicalType + "\"}}]}");
+ HoodieSchema hoodieSchema = HoodieSchema.fromAvroSchema(schema);
+ Instant instant = Instant.parse("2026-10-01T00:00:00.123456Z");
+ LocalDateTime localTime = LocalDateTime.ofInstant(instant, ZoneOffset.UTC);
+ long rawValue = logicalType.equals("local-timestamp-millis") ?
instant.toEpochMilli()
+ : instant.getEpochSecond() * 1_000_000 + instant.getNano() / 1_000;
+ GenericRecord record = new GenericData.Record(schema);
+ record.put("ts", rawValue);
+ TypedProperties props = new TypedProperties();
+
props.setProperty(KeyGeneratorOptions.KEYGENERATOR_CONSISTENT_LOGICAL_TIMESTAMP_ENABLED.key(),
"true");
+ AvroRecordContext context = new AvroRecordContext(new HoodieTableConfig(),
null, props);
+ List<String> fields = Collections.singletonList("ts");
+ context.initOrderingValueConverter(hoodieSchema, fields);
+ Comparable expected = context.getOrderingValue(record, hoodieSchema,
fields);
+ assertEquals(expected,
context.getOrderingValue(roundTripDelete(rawValue)));
+ assertEquals(expected,
context.getOrderingValue(roundTripDelete(Timestamp.valueOf(localTime))));
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void testTimestampDeleteOrderingPrecision(boolean consistentTimestamp) {
+ TypedProperties props = new TypedProperties();
+
props.setProperty(KeyGeneratorOptions.KEYGENERATOR_CONSISTENT_LOGICAL_TIMESTAMP_ENABLED.key(),
String.valueOf(consistentTimestamp));
+ AvroRecordContext context = new AvroRecordContext(new HoodieTableConfig(),
null, props);
+ for (String logicalType : Arrays.asList("timestamp-millis",
"timestamp-micros", "local-timestamp-millis", "local-timestamp-micros")) {
+ Schema schema = new
Schema.Parser().parse("{\"type\":\"record\",\"name\":\"timestamp_ordering\",\"fields\":["
+ + "{\"name\":\"ts\",\"type\":{\"type\":\"long\",\"logicalType\":\""
+ logicalType + "\"}}]}");
+ context.initOrderingValueConverter(HoodieSchema.fromAvroSchema(schema),
Collections.singletonList("ts"));
+ for (long micros : new long[] {-1001L, 1790812800123456L}) {
+ Instant instant = Instant.ofEpochSecond(Math.floorDiv(micros,
1_000_000L), Math.floorMod(micros, 1_000_000L) * 1000L);
+ long rawValue = logicalType.endsWith("millis") ?
instant.toEpochMilli() : micros;
+ Object expected =
HoodieAvroUtils.convertValueForAvroLogicalTypes(schema.getField("ts").schema(),
rawValue, consistentTimestamp);
+ // Local timestamps preserve wall-clock fields rather than the instant
in the JVM's timezone.
+ Timestamp timestamp = logicalType.startsWith("local-")
+ ? Timestamp.valueOf(LocalDateTime.ofInstant(instant,
ZoneOffset.UTC)) : Timestamp.from(instant);
+ assertEquals(expected,
context.convertOrderingValueToEngineType(timestamp), logicalType);
+ assertEquals(expected,
context.convertOrderingValueToEngineType(rawValue), logicalType);
+ Comparable javaTimeValue = logicalType.startsWith("local-") ?
LocalDateTime.ofInstant(instant, ZoneOffset.UTC) : instant;
+ assertEquals(expected,
context.convertOrderingValueToEngineType(javaTimeValue), logicalType);
+ }
+ }
+ }
+
+ @Test
+ void testMultiFieldDeleteOrderingWithEpochDate() throws IOException {
+ HoodieSchema schema = HoodieSchema.fromAvroSchema(new
Schema.Parser().parse(
+ "{\"type\":\"record\",\"name\":\"epoch_date\",\"fields\":["
+ + "{\"name\":\"rank\",\"type\":\"int\"},"
+ +
"{\"name\":\"date\",\"type\":{\"type\":\"int\",\"logicalType\":\"date\"}}]}"));
+ GenericRecord record = new GenericData.Record(schema.toAvroSchema());
+ record.put("rank", 1);
+ record.put("date", 0);
+ List<String> fields = Arrays.asList("rank", "date");
+ AvroRecordContext context = new AvroRecordContext();
+ context.initOrderingValueConverter(schema, fields);
+
+ Comparable dataOrdering = context.getOrderingValue(record, schema, fields);
+ Comparable deleteOrdering = context.getOrderingValue(roundTripDelete(new
ArrayComparable(new Comparable[] {1, 0})));
+ assertEquals(0, dataOrdering.compareTo(deleteOrdering));
+ }
+
+ @Test
+ void testDeleteOrderingPreservesSentinelsAndValidatesFields() {
+ HoodieSchema schema = HoodieSchema.fromAvroSchema(new
Schema.Parser().parse(
+ "{\"type\":\"record\",\"name\":\"sentinels\",\"fields\":["
+ +
"{\"name\":\"date\",\"type\":{\"type\":\"int\",\"logicalType\":\"date\"}},"
+ +
"{\"name\":\"ts\",\"type\":{\"type\":\"long\",\"logicalType\":\"timestamp-micros\"}}]}"));
+ AvroRecordContext context = new AvroRecordContext();
+ context.initOrderingValueConverter(schema, Arrays.asList("date", "ts"));
+ assertEquals(OrderingValues.getDefault(),
context.getOrderingValue(DeleteRecord.create("key", "", null)));
+ assertEquals(OrderingValues.getDefault(),
context.getOrderingValue(DeleteRecord.create("key", "",
OrderingValues.getDefault())));
+ assertNull(context.convertOrderingValueToEngineType(null));
+ assertEquals(LocalDate.ofEpochDay(0),
context.convertOrderingValueToEngineType(0));
+ ArrayComparable values = new ArrayComparable(new Comparable[] {0, null});
+ ArrayComparable expected = new ArrayComparable(new Comparable[]
{LocalDate.ofEpochDay(0), null});
+ assertEquals(expected, context.getOrderingValue(DeleteRecord.create("key",
"", values)));
+
+ IllegalArgumentException exception =
assertThrows(IllegalArgumentException.class,
+ () -> context.initOrderingValueConverter(schema,
Collections.singletonList("missing")));
+ assertEquals("Ordering field 'missing' not found in table schema",
exception.getMessage());
+ context.initOrderingValueConverter(schema, Collections.emptyList());
+ assertEquals("value", context.getOrderingValue(DeleteRecord.create("key",
"", "value")));
+ }
+
+ private static DeleteRecord roundTripDelete(Comparable value) throws
IOException {
+ HoodieDeleteBlock written = new HoodieDeleteBlock(
+ Collections.singletonList(Pair.of(DeleteRecord.create("key", "",
value), -1L)), Collections.emptyMap());
+ HoodieDeleteBlock read = new
HoodieDeleteBlock(Option.of(written.getContentBytes(null).toByteArray()),
+ null, false, Option.empty(), Collections.emptyMap(),
Collections.emptyMap());
+ return read.getRecordsToDelete()[0];
+ }
+
private static Stream<Arguments> testConvertValueToEngineType() {
return Stream.of(
Arguments.of(1L, 1L),
@@ -96,7 +327,7 @@ class TestAvroRecordContext {
if (partitionFields != null) {
tableConfig.setValue(HoodieTableConfig.PARTITION_FIELDS,
partitionFields);
}
- AvroRecordContext context = new AvroRecordContext(tableConfig, null);
+ AvroRecordContext context = new AvroRecordContext(tableConfig, null, new
TypedProperties());
assertEquals(expectedKey, context.getRecordKey(buildRecord(),
HoodieSchema.fromAvroSchema(RECORD_SCHEMA)));
}
diff --git
a/hudi-common/src/test/java/org/apache/hudi/common/avro/TestHoodieAvroReaderContext.java
b/hudi-common/src/test/java/org/apache/hudi/common/avro/TestHoodieAvroReaderContext.java
index 927158d890dd..2c59f76ea4c7 100644
---
a/hudi-common/src/test/java/org/apache/hudi/common/avro/TestHoodieAvroReaderContext.java
+++
b/hudi-common/src/test/java/org/apache/hudi/common/avro/TestHoodieAvroReaderContext.java
@@ -20,10 +20,12 @@
package org.apache.hudi.common.avro;
import org.apache.hudi.common.model.DefaultHoodieRecordPayload;
+import org.apache.hudi.common.model.DeleteRecord;
import org.apache.hudi.common.schema.HoodieSchema;
import org.apache.hudi.common.schema.HoodieSchemaUtils;
import org.apache.hudi.common.table.HoodieTableConfig;
import org.apache.hudi.common.table.read.BufferedRecord;
+import org.apache.hudi.common.table.read.FileGroupReaderSchemaHandler;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.ClosableIterator;
import org.apache.hudi.storage.StorageConfiguration;
@@ -35,6 +37,7 @@ import org.apache.avro.generic.IndexedRecord;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import java.time.LocalDate;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -62,6 +65,28 @@ class TestHoodieAvroReaderContext {
@BeforeEach
void setup() {
when(tableConfig.getPayloadClass()).thenReturn(DefaultHoodieRecordPayload.class.getName());
+ when(tableConfig.getPayloadClassIfPresent()).thenReturn(Option.empty());
+ }
+
+ @Test
+ void initOrderingValueConverterUsesTableSchema() {
+ HoodieSchema fullSchema = HoodieSchema.fromAvroSchema(new
Schema.Parser().parse(
+ "{\"type\":\"record\",\"name\":\"table_record\",\"fields\":["
+ + "{\"name\":\"id\",\"type\":\"string\"},"
+ +
"{\"name\":\"date\",\"type\":{\"type\":\"int\",\"logicalType\":\"date\"}}]}"));
+ HoodieSchema projectedSchema = HoodieSchema.fromAvroSchema(new
Schema.Parser().parse(
+
"{\"type\":\"record\",\"name\":\"table_record\",\"fields\":[{\"name\":\"id\",\"type\":\"string\"}]}"));
+
when(tableConfig.getOrderingFields()).thenReturn(Collections.singletonList("date"));
+ HoodieAvroReaderContext readerContext = getReaderContextWithMetaFields();
+ readerContext.setHasLogFiles(false);
+ FileGroupReaderSchemaHandler<IndexedRecord> schemaHandler =
mock(FileGroupReaderSchemaHandler.class);
+ when(schemaHandler.getTableSchema()).thenReturn(fullSchema);
+ // Base-file-only reads may project away the ordering field.
+ when(schemaHandler.getRequiredSchema()).thenReturn(projectedSchema);
+ readerContext.setSchemaHandler(schemaHandler);
+
+ assertEquals(LocalDate.ofEpochDay(10),
+
readerContext.getRecordContext().getOrderingValue(DeleteRecord.create("key",
"", 10)));
}
@Test
diff --git
a/hudi-common/src/test/java/org/apache/hudi/common/serialization/TestBufferedRecordSerializer.java
b/hudi-common/src/test/java/org/apache/hudi/common/serialization/TestBufferedRecordSerializer.java
index 0f189be7b6d8..8dc6b4ff5150 100644
---
a/hudi-common/src/test/java/org/apache/hudi/common/serialization/TestBufferedRecordSerializer.java
+++
b/hudi-common/src/test/java/org/apache/hudi/common/serialization/TestBufferedRecordSerializer.java
@@ -19,9 +19,7 @@
package org.apache.hudi.common.serialization;
import org.apache.hudi.avro.model.HoodieMetadataRecord;
-import org.apache.hudi.common.avro.AvroRecordContext;
import org.apache.hudi.common.avro.AvroRecordSerializer;
-import org.apache.hudi.common.model.DeleteRecord;
import org.apache.hudi.common.model.HoodieOperation;
import org.apache.hudi.common.schema.HoodieSchema;
import org.apache.hudi.common.table.read.BufferedRecord;
@@ -101,8 +99,7 @@ public class TestBufferedRecordSerializer {
@EnumSource(value = HoodieOperation.class, names = {"UPDATE_BEFORE",
"DELETE"})
void testDeleteRecordSerAndDe(HoodieOperation hoodieOperation) throws
IOException {
HoodieSchema schema = SchemaTestUtil.getSimpleSchema();
- DeleteRecord record = DeleteRecord.create("id", "partition", 100);
- BufferedRecord<IndexedRecord> bufferedRecord =
BufferedRecords.fromDeleteRecord(record,
AvroRecordContext.getFieldAccessorInstance());
+ BufferedRecord<IndexedRecord> bufferedRecord =
BufferedRecords.createDelete("id", 100);
bufferedRecord.setHoodieOperation(hoodieOperation);
AvroRecordSerializer avroRecordSerializer = new
AvroRecordSerializer(integer -> schema.toAvroSchema());
diff --git
a/hudi-common/src/test/java/org/apache/hudi/common/table/read/TestBufferedRecordMergerFactory.java
b/hudi-common/src/test/java/org/apache/hudi/common/table/read/TestBufferedRecordMergerFactory.java
index 8015d0dcd8f9..2db89501d46d 100644
---
a/hudi-common/src/test/java/org/apache/hudi/common/table/read/TestBufferedRecordMergerFactory.java
+++
b/hudi-common/src/test/java/org/apache/hudi/common/table/read/TestBufferedRecordMergerFactory.java
@@ -23,6 +23,7 @@ import org.apache.hudi.common.config.RecordMergeMode;
import org.apache.hudi.common.config.TypedProperties;
import org.apache.hudi.common.engine.HoodieReaderContext;
import org.apache.hudi.common.engine.RecordContext;
+import org.apache.hudi.common.model.DeleteRecord;
import org.apache.hudi.common.model.HoodieRecordMerger;
import org.apache.hudi.common.schema.HoodieSchema;
import org.apache.hudi.common.schema.HoodieSchemaType;
@@ -32,7 +33,12 @@ import org.apache.hudi.common.util.collection.Pair;
import org.junit.jupiter.api.Test;
+import java.io.IOException;
+import java.time.LocalDate;
+
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.CALLS_REAL_METHODS;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -68,6 +74,25 @@ class TestBufferedRecordMergerFactory {
.getClass().getSimpleName());
}
+ @Test
+ void testLegacyDeleteMergePreservesSentinelsBeforeConversion() throws
IOException {
+ RecordContext<String> recordContext = mock(RecordContext.class,
CALLS_REAL_METHODS);
+ when(recordContext.convertOrderingValueToEngineType(any())).thenAnswer(
+ invocation -> LocalDate.ofEpochDay(((Number)
invocation.getArgument(0)).longValue()));
+ HoodieReaderContext<String> readerContext =
mock(HoodieReaderContext.class);
+ when(readerContext.getRecordContext()).thenReturn(recordContext);
+ BufferedRecordMerger<String> merger = BufferedRecordMergerFactory.create(
+ readerContext, RecordMergeMode.EVENT_TIME_ORDERING, false,
Option.empty(),
+ HoodieSchema.create(HoodieSchemaType.STRING), Option.empty(), new
TypedProperties(), Option.empty());
+ BufferedRecord<String> existing = new BufferedRecord<>("key",
LocalDate.ofEpochDay(10), "record", 0, null);
+
+ for (Comparable orderingValue : new Comparable[] {null, 0}) {
+ DeleteRecord delete = DeleteRecord.create("key", "", orderingValue);
+ assertEquals(Option.of(delete), merger.deltaMerge(delete, existing));
+ }
+ assertEquals(Option.empty(), merger.deltaMerge(DeleteRecord.create("key",
"", 9), existing));
+ }
+
private static void assertMerger(String expected,
HoodieReaderContext<String> context,
RecordMergeMode mode,
diff --git
a/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestFileGroupRecordBufferLoader.java
b/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestFileGroupRecordBufferLoader.java
index 16a2eb893f95..f639aaecdc92 100644
---
a/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestFileGroupRecordBufferLoader.java
+++
b/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestFileGroupRecordBufferLoader.java
@@ -63,6 +63,7 @@ public class TestFileGroupRecordBufferLoader extends
BaseTestFileGroupRecordBuff
: FileGroupRecordBufferLoader.createStreamingRecordsBufferLoader();
HoodieReadStats readStats = new HoodieReadStats();
HoodieTableConfig tableConfig = mock(HoodieTableConfig.class);
+ when(tableConfig.getPayloadClassIfPresent()).thenReturn(Option.empty());
when(tableConfig.getRecordMergeMode()).thenReturn(RecordMergeMode.COMMIT_TIME_ORDERING);
when(tableConfig.getTableVersion()).thenReturn(HoodieTableVersion.NINE);
when(tableConfig.getPartialUpdateMode()).thenReturn(Option.empty());
diff --git
a/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestKeyBasedFileGroupRecordBuffer.java
b/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestKeyBasedFileGroupRecordBuffer.java
index 60c0552a444a..d9a6df1fc4f5 100644
---
a/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestKeyBasedFileGroupRecordBuffer.java
+++
b/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestKeyBasedFileGroupRecordBuffer.java
@@ -21,21 +21,32 @@ package org.apache.hudi.common.table.read.buffer;
import org.apache.hudi.common.avro.HoodieAvroReaderContext;
import org.apache.hudi.common.config.RecordMergeMode;
+import org.apache.hudi.common.config.TypedProperties;
import org.apache.hudi.common.engine.HoodieReaderContext;
import org.apache.hudi.common.model.DeleteRecord;
import org.apache.hudi.common.model.HoodieAvroRecordMerger;
+import org.apache.hudi.common.schema.HoodieSchema;
+import org.apache.hudi.common.schema.HoodieSchemaField;
+import org.apache.hudi.common.schema.HoodieSchemaType;
+import org.apache.hudi.common.schema.internal.InternalSchema;
import org.apache.hudi.common.table.HoodieTableConfig;
import org.apache.hudi.common.table.log.block.HoodieDataBlock;
import org.apache.hudi.common.table.log.block.HoodieDeleteBlock;
import org.apache.hudi.common.table.read.BufferedRecord;
+import org.apache.hudi.common.table.read.DeleteContext;
+import org.apache.hudi.common.table.read.FileGroupReaderSchemaHandler;
import org.apache.hudi.common.table.read.HoodieReadStats;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.ClosableIterator;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericRecord;
import org.apache.avro.generic.IndexedRecord;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
import java.io.IOException;
import java.io.Serializable;
@@ -47,6 +58,7 @@ import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -93,6 +105,7 @@ class TestKeyBasedFileGroupRecordBuffer extends
BaseTestFileGroupRecordBuffer {
void readWithEventTimeOrdering() throws IOException {
HoodieReadStats readStats = new HoodieReadStats();
HoodieTableConfig tableConfig = mock(HoodieTableConfig.class);
+ when(tableConfig.getPayloadClassIfPresent()).thenReturn(Option.empty());
when(tableConfig.getRecordKeyFields()).thenReturn(Option.of(new String[]
{"record_key"}));
when(tableConfig.getPartitionFields()).thenReturn(Option.empty());
StorageConfiguration<?> storageConfiguration =
mock(StorageConfiguration.class);
@@ -121,6 +134,7 @@ class TestKeyBasedFileGroupRecordBuffer extends
BaseTestFileGroupRecordBuffer {
void readWithEventTimeOrderingAndDeleteBlock() throws IOException {
HoodieReadStats readStats = new HoodieReadStats();
HoodieTableConfig tableConfig = mock(HoodieTableConfig.class);
+ when(tableConfig.getPayloadClassIfPresent()).thenReturn(Option.empty());
when(tableConfig.getRecordKeyFields()).thenReturn(Option.of(new String[]
{"record_key"}));
when(tableConfig.getPartitionFields()).thenReturn(Option.empty());
StorageConfiguration<?> storageConfiguration =
mock(StorageConfiguration.class);
@@ -152,10 +166,77 @@ class TestKeyBasedFileGroupRecordBuffer extends
BaseTestFileGroupRecordBuffer {
assertEquals(2, readStats.getNumUpdates());
}
+ @ParameterizedTest
+ @CsvSource({
+ "9, false, false",
+ "9, true, false",
+ "10, false, true",
+ "10, true, false",
+ "11, false, true",
+ "11, true, true"
+ })
+ void readWithDateOrderingAndSerializedDeleteBlock(int deleteEpochDay,
boolean deleteFirst, boolean expectDeleted) throws IOException {
+ HoodieSchema dateSchema = HoodieSchema.createRecord("date_record",
"namespace", null,
+ Arrays.asList(
+ HoodieSchemaField.of("record_key",
HoodieSchema.create(HoodieSchemaType.STRING)),
+ HoodieSchemaField.of("ts",
HoodieSchema.create(HoodieSchemaType.DATE))));
+ GenericRecord record = new GenericData.Record(dateSchema.toAvroSchema());
+ record.put("record_key", "1");
+ record.put("ts", 10);
+
+ TypedProperties props = new TypedProperties();
+ props.setProperty(HoodieTableConfig.ORDERING_FIELDS.key(), "ts");
+ HoodieTableConfig tableConfig = mock(HoodieTableConfig.class);
+ when(tableConfig.getPayloadClassIfPresent()).thenReturn(Option.empty());
+ when(tableConfig.getRecordKeyFields()).thenReturn(Option.of(new String[]
{"record_key"}));
+ when(tableConfig.getPartitionFields()).thenReturn(Option.empty());
+
when(tableConfig.getOrderingFields()).thenReturn(Collections.singletonList("ts"));
+ HoodieReaderContext<IndexedRecord> readerContext = new
HoodieAvroReaderContext(
+ mock(StorageConfiguration.class), tableConfig, Option.empty(),
Option.empty(), props);
+ FileGroupReaderSchemaHandler<IndexedRecord> schemaHandler =
mock(FileGroupReaderSchemaHandler.class);
+ when(schemaHandler.getTableSchema()).thenReturn(dateSchema);
+ when(schemaHandler.getRequiredSchema()).thenReturn(dateSchema);
+ when(schemaHandler.getSchemaForUpdates()).thenReturn(dateSchema);
+
when(schemaHandler.getInternalSchema()).thenReturn(InternalSchema.getEmptyInternalSchema());
+ when(schemaHandler.getSchemaEvolutionTransformer(any(),
any())).thenReturn(Option.empty());
+ when(schemaHandler.getDeleteContext()).thenReturn(new DeleteContext(props,
dateSchema));
+ readerContext.setSchemaHandler(schemaHandler);
+
+ HoodieDataBlock dataBlock = mock(HoodieDataBlock.class);
+ when(dataBlock.getSchema()).thenReturn(dateSchema);
+ when(dataBlock.getEngineRecordIterator(readerContext))
+
.thenReturn(ClosableIterator.wrap(Collections.<IndexedRecord>singletonList(record).iterator()));
+ // Inline delete blocks retain the physical integer even though Avro DATE
ordering is a LocalDate.
+ HoodieDeleteBlock writtenDeleteBlock = new HoodieDeleteBlock(
+ Collections.singletonList(Pair.of(DeleteRecord.create("1", "",
deleteEpochDay), -1L)), Collections.emptyMap());
+ HoodieDeleteBlock deleteBlock = new
HoodieDeleteBlock(Option.of(writtenDeleteBlock.getContentBytes(null).toByteArray()),
+ null, false, Option.empty(), Collections.emptyMap(),
Collections.emptyMap());
+ assertEquals(Integer.valueOf(deleteEpochDay),
deleteBlock.getRecordsToDelete()[0].getOrderingValue());
+
+ KeyBasedFileGroupRecordBuffer<IndexedRecord> buffer =
buildKeyBasedFileGroupRecordBuffer(
+ readerContext, tableConfig, new HoodieReadStats(), null,
RecordMergeMode.EVENT_TIME_ORDERING,
+ Collections.singletonList("ts"), props, Option.empty());
+ try {
+
buffer.setBaseFileIterator(ClosableIterator.wrap(Collections.<IndexedRecord>emptyList().iterator()));
+ if (deleteFirst) {
+ buffer.processDeleteBlock(deleteBlock);
+ buffer.processDataBlock(dataBlock, Option.empty());
+ } else {
+ buffer.processDataBlock(dataBlock, Option.empty());
+ buffer.processDeleteBlock(deleteBlock);
+ }
+ // An older delete loses and a newer delete wins in either order; equal
event times keep the last arrival.
+ assertEquals(expectDeleted ? Collections.emptyList() :
Collections.singletonList(record), getActualRecords(buffer));
+ } finally {
+ buffer.close();
+ }
+ }
+
@Test
void readWithCommitTimeOrdering() throws IOException {
HoodieReadStats readStats = new HoodieReadStats();
HoodieTableConfig tableConfig = mock(HoodieTableConfig.class);
+ when(tableConfig.getPayloadClassIfPresent()).thenReturn(Option.empty());
when(tableConfig.getRecordKeyFields()).thenReturn(Option.of(new String[]
{"record_key"}));
when(tableConfig.getPartitionFields()).thenReturn(Option.empty());
StorageConfiguration<?> storageConfiguration =
mock(StorageConfiguration.class);
@@ -187,6 +268,7 @@ class TestKeyBasedFileGroupRecordBuffer extends
BaseTestFileGroupRecordBuffer {
void readWithCustomPayload() throws IOException {
HoodieReadStats readStats = new HoodieReadStats();
HoodieTableConfig tableConfig = mock(HoodieTableConfig.class);
+ when(tableConfig.getPayloadClassIfPresent()).thenReturn(Option.empty());
when(tableConfig.getPayloadClass()).thenReturn(CustomPayload.class.getName());
when(tableConfig.getRecordKeyFields()).thenReturn(Option.of(new String[]
{"record_key"}));
when(tableConfig.getPartitionFields()).thenReturn(Option.empty());
@@ -227,6 +309,7 @@ class TestKeyBasedFileGroupRecordBuffer extends
BaseTestFileGroupRecordBuffer {
void readWithCustomMerger() throws IOException {
HoodieReadStats readStats = new HoodieReadStats();
HoodieTableConfig tableConfig = mock(HoodieTableConfig.class);
+ when(tableConfig.getPayloadClassIfPresent()).thenReturn(Option.empty());
when(tableConfig.getPayloadClass()).thenReturn(CustomPayload.class.getName());
when(tableConfig.getRecordKeyFields()).thenReturn(Option.of(new String[]
{"record_key"}));
when(tableConfig.getPartitionFields()).thenReturn(Option.empty());
diff --git
a/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestSortedKeyBasedFileGroupRecordBuffer.java
b/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestSortedKeyBasedFileGroupRecordBuffer.java
index d1418101c13e..a8bbfa08beb6 100644
---
a/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestSortedKeyBasedFileGroupRecordBuffer.java
+++
b/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestSortedKeyBasedFileGroupRecordBuffer.java
@@ -127,6 +127,7 @@ class TestSortedKeyBasedFileGroupRecordBuffer extends
BaseTestFileGroupRecordBuf
properties.setProperty(DELETE_KEY, "counter");
properties.setProperty(DELETE_MARKER, "3");
HoodieTableConfig tableConfig = mock(HoodieTableConfig.class);
+ when(tableConfig.getPayloadClassIfPresent()).thenReturn(Option.empty());
when(tableConfig.getRecordKeyFields()).thenReturn(Option.of(new String[]
{"record_key"}));
when(tableConfig.getPartitionFields()).thenReturn(Option.empty());
when(tableConfig.getTableVersion()).thenReturn(HoodieTableVersion.current());
diff --git
a/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestStreamingKeyBasedFileGroupRecordBuffer.java
b/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestStreamingKeyBasedFileGroupRecordBuffer.java
index d03be553194a..5e652d7965ce 100644
---
a/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestStreamingKeyBasedFileGroupRecordBuffer.java
+++
b/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestStreamingKeyBasedFileGroupRecordBuffer.java
@@ -74,6 +74,7 @@ class TestStreamingKeyBasedFileGroupRecordBuffer extends
BaseTestFileGroupRecord
properties.setProperty(DELETE_KEY, "counter");
properties.setProperty(DELETE_MARKER, "3");
HoodieTableConfig tableConfig = mock(HoodieTableConfig.class);
+ when(tableConfig.getPayloadClassIfPresent()).thenReturn(Option.empty());
when(tableConfig.getRecordMergeMode()).thenReturn(RecordMergeMode.EVENT_TIME_ORDERING);
when(tableConfig.getPartialUpdateMode()).thenReturn(Option.empty());
when(tableConfig.getTableVersion()).thenReturn(HoodieTableVersion.current());
@@ -113,6 +114,7 @@ class TestStreamingKeyBasedFileGroupRecordBuffer extends
BaseTestFileGroupRecord
properties.setProperty(DELETE_KEY, "counter");
properties.setProperty(DELETE_MARKER, "3");
HoodieTableConfig tableConfig = mock(HoodieTableConfig.class);
+ when(tableConfig.getPayloadClassIfPresent()).thenReturn(Option.empty());
when(tableConfig.getRecordMergeMode()).thenReturn(RecordMergeMode.COMMIT_TIME_ORDERING);
when(tableConfig.getPartialUpdateMode()).thenReturn(Option.empty());
when(tableConfig.getTableVersion()).thenReturn(HoodieTableVersion.current());
@@ -152,6 +154,7 @@ class TestStreamingKeyBasedFileGroupRecordBuffer extends
BaseTestFileGroupRecord
properties.setProperty(HoodieTableConfig.PAYLOAD_CLASS_NAME.key(),
CustomPayload.class.getName());
properties.setProperty(HoodieTableConfig.RECORD_MERGE_STRATEGY_ID.key(),
HoodieRecordMerger.PAYLOAD_BASED_MERGE_STRATEGY_UUID);
HoodieTableConfig tableConfig = mock(HoodieTableConfig.class);
+ when(tableConfig.getPayloadClassIfPresent()).thenReturn(Option.empty());
when(tableConfig.getPayloadClass()).thenReturn(CustomPayload.class.getName());
when(tableConfig.getRecordKeyFields()).thenReturn(Option.of(new String[]
{"record_key"}));
when(tableConfig.getPartitionFields()).thenReturn(Option.empty());
@@ -190,6 +193,7 @@ class TestStreamingKeyBasedFileGroupRecordBuffer extends
BaseTestFileGroupRecord
properties.setProperty(DELETE_MARKER, "3");
properties.setProperty(HoodieTableConfig.PAYLOAD_CLASS_NAME.key(),
CustomPayload.class.getName());
HoodieTableConfig tableConfig = mock(HoodieTableConfig.class);
+ when(tableConfig.getPayloadClassIfPresent()).thenReturn(Option.empty());
when(tableConfig.getPayloadClass()).thenReturn(CustomPayload.class.getName());
when(tableConfig.getRecordKeyFields()).thenReturn(Option.of(new String[]
{"record_key"}));
when(tableConfig.getPartitionFields()).thenReturn(Option.empty());
diff --git
a/hudi-common/src/test/java/org/apache/hudi/metadata/TestHoodieBackedTableMetadataDataCleanup.java
b/hudi-common/src/test/java/org/apache/hudi/metadata/TestHoodieBackedTableMetadataDataCleanup.java
index 27a49bf897f7..0854fcea85a9 100644
---
a/hudi-common/src/test/java/org/apache/hudi/metadata/TestHoodieBackedTableMetadataDataCleanup.java
+++
b/hudi-common/src/test/java/org/apache/hudi/metadata/TestHoodieBackedTableMetadataDataCleanup.java
@@ -602,6 +602,7 @@ public class TestHoodieBackedTableMetadataDataCleanup {
when(timeline.filterCompletedInstants()).thenReturn(timeline);
when(timeline.lastInstant()).thenReturn(Option.empty());
HoodieTableConfig tableConfig = mock(HoodieTableConfig.class);
+ when(tableConfig.getPayloadClassIfPresent()).thenReturn(Option.empty());
when(tableConfig.isRecordKeyPopulated()).thenReturn(true);
when(tableConfig.getBaseFileFormat()).thenReturn(HoodieFileFormat.PARQUET);
when(metadataMetaClient.getTableConfig()).thenReturn(tableConfig);
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBufferedRecordMerger.java
b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBufferedRecordMerger.java
index d074cff217ff..65a6ecea2e0b 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBufferedRecordMerger.java
+++
b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBufferedRecordMerger.java
@@ -617,6 +617,7 @@ class TestBufferedRecordMerger extends
SparkClientFunctionalTestHarness {
payloadClassName = Option.empty();
}
// Configure reader context with custom schema
+ when(tableConfig.getPayloadClassIfPresent()).thenReturn(Option.empty());
props.setProperty(ORDERING_FIELDS.key(), "timestamp");
HoodieAvroReaderContext avroReaderContext = new
HoodieAvroReaderContext(storageConfig, tableConfig, Option.empty(),
Option.empty());
avroReaderContext.setHasLogFiles(false);
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/others/TestMergeModeEventTimeOrdering.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/others/TestMergeModeEventTimeOrdering.scala
index 1c61a85fcf81..e898afc7d4ab 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/others/TestMergeModeEventTimeOrdering.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/others/TestMergeModeEventTimeOrdering.scala
@@ -23,6 +23,7 @@ import org.apache.hudi.DataSourceWriteOptions
import org.apache.hudi.DataSourceWriteOptions.SPARK_SQL_INSERT_INTO_OPERATION
import org.apache.hudi.common.config.RecordMergeMode.EVENT_TIME_ORDERING
import org.apache.hudi.common.model.DefaultHoodieRecordPayload
+import org.apache.hudi.common.model.HoodieRecord.HoodieRecordType
import
org.apache.hudi.common.model.HoodieRecordMerger.EVENT_TIME_BASED_MERGE_STRATEGY_UUID
import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableVersion}
import org.apache.hudi.common.testutils.HoodieTestUtils
@@ -31,6 +32,36 @@ import
org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase
import
org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase.validateTableConfig
class TestMergeModeEventTimeOrdering extends HoodieSparkSqlTestBase {
+ Seq(
+ ("date", "date'2026-10-01'", "date'2026-10-02'", false),
+ ("timestamp", "timestamp'2026-10-01 00:00:00'", "timestamp'2026-10-02
00:00:00'", false),
+ ("timestamp", "timestamp'2026-10-01 00:00:00'", "timestamp'2026-10-02
00:00:00'", true),
+ ("decimal(10,2)", "cast(1.50 as decimal(10,2))", "cast(2.50 as
decimal(10,2))", false)
+ ).foreach { case (orderingType, oldValue, newValue, consistentTimestamp) =>
+ test(s"COW SQL upsert with $orderingType ordering field
(consistentTimestamp=$consistentTimestamp)") {
+ withSQLConf(SPARK_SQL_INSERT_INTO_OPERATION.key -> "upsert",
+
"hoodie.datasource.write.keygenerator.consistent.logical.timestamp.enabled" ->
consistentTimestamp.toString) {
+ withRecordType(Seq(HoodieRecordType.AVRO)) {
+ withTempDir { tmp =>
+ val tableName = generateTableName
+ spark.sql(
+ s"""
+ | create table $tableName (id int, v int, ts $orderingType)
+ | using hudi
+ | tblproperties (type = 'cow', primaryKey = 'id',
preCombineField = 'ts')
+ | location '${tmp.getCanonicalPath}'
+ """.stripMargin)
+ spark.sql(s"insert into $tableName values (1, 1, $oldValue)")
+ spark.sql(s"insert into $tableName values (1, 2, $newValue)")
+ checkAnswer(s"select id, v from $tableName")(Seq(1, 2))
+ spark.sql(s"insert into $tableName values (1, 3, $oldValue)")
+ checkAnswer(s"select id, v from $tableName")(Seq(1, 2))
+ }
+ }
+ }
+ }
+ }
+
Seq(
"cow,current,true",
"cow,current,false",
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/others/TestUpdateTable.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/others/TestUpdateTable.scala
index bee62c001858..9a63eced1354 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/others/TestUpdateTable.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/others/TestUpdateTable.scala
@@ -34,6 +34,53 @@ import org.junit.jupiter.api.Assertions.assertEquals
class TestUpdateTable extends HoodieSparkSqlTestBase {
+ // (ordering field type, older literal, newer literal, older value as
string, newer value as string)
+ private val logicalTypeOrderingCases = Seq(
+ ("timestamp", "timestamp'2026-10-01 00:00:00'", "timestamp'2026-10-02
00:00:00'",
+ "2026-10-01 00:00:00", "2026-10-02 00:00:00"),
+ ("date", "date'2026-10-01'", "date'2026-10-02'", "2026-10-01",
"2026-10-02"),
+ ("decimal(10,2)", "cast(1.50 as decimal(10,2))", "cast(2.50 as
decimal(10,2))", "1.50", "2.50"))
+
+ logicalTypeOrderingCases.foreach { case (orderingType, oldLiteral,
newLiteral, oldStr, newStr) =>
+ test(s"Test Update Table With ${orderingType.toUpperCase} Ordering Field")
{
+ withTempDir { tmp =>
+ Seq(true, false).foreach { sparkSqlOptimizedWrites =>
+ val tableName = generateTableName
+ spark.sql(
+ s"""
+ |create table $tableName (
+ | id int,
+ | v int,
+ | ts $orderingType,
+ | dt string
+ |) using hudi
+ | location '${tmp.getCanonicalPath}/$tableName'
+ | tblproperties (
+ | type = 'cow',
+ | primaryKey = 'id',
+ | preCombineField = 'ts'
+ | )
+ """.stripMargin)
+ spark.sql(s"insert into $tableName values (1, 1, $oldLiteral, 'p1'),
(2, 2, $oldLiteral, 'p1')")
+
+ withSQLConf(SPARK_SQL_OPTIMIZED_WRITES.key() ->
sparkSqlOptimizedWrites.toString) {
+ spark.sql(s"update $tableName set v = v * 10 where id = 1")
+ checkAnswer(s"select id, v, cast(ts as string), dt from $tableName
order by id")(
+ Seq(1, 10, oldStr, "p1"),
+ Seq(2, 2, oldStr, "p1")
+ )
+
+ spark.sql(s"update $tableName set v = v + 1, ts = $newLiteral
where id = 2")
+ checkAnswer(s"select id, v, cast(ts as string), dt from $tableName
order by id")(
+ Seq(1, 10, oldStr, "p1"),
+ Seq(2, 3, newStr, "p1")
+ )
+ }
+ }
+ }
+ }
+ }
+
test("Test Update Table") {
withRecordType()(withTempDir { tmp =>
Seq(true, false).foreach { sparkSqlOptimizedWrites =>
diff --git
a/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiPageSourceProvider.java
b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiPageSourceProvider.java
index aafc7ea1c5ff..cee3c74dba07 100644
--- a/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiPageSourceProvider.java
+++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiPageSourceProvider.java
@@ -279,7 +279,8 @@ public class HudiPageSourceProvider
dataPageSource,
readColumnHandles,
prefilledColumnValues,
- logPageSourceFactory);
+ logPageSourceFactory,
+ readerProps);
Schema requestedSchema = constructSchema(dataSchema.toAvroSchema(),
readColumnHandles.stream().map(HiveColumnHandle::getName).toList());
FileSlice fileSlice = convertToFileSlice(hudiSplit,
hudiTableHandle.getBasePath());
diff --git
a/hudi-trino/src/main/java/io/trino/plugin/hudi/reader/HudiTrinoReaderContext.java
b/hudi-trino/src/main/java/io/trino/plugin/hudi/reader/HudiTrinoReaderContext.java
index cc812a9a466f..36212bf21d29 100644
---
a/hudi-trino/src/main/java/io/trino/plugin/hudi/reader/HudiTrinoReaderContext.java
+++
b/hudi-trino/src/main/java/io/trino/plugin/hudi/reader/HudiTrinoReaderContext.java
@@ -25,6 +25,7 @@ import io.trino.spi.connector.SourcePage;
import org.apache.avro.generic.IndexedRecord;
import org.apache.hudi.common.avro.AvroRecordContext;
import org.apache.hudi.common.config.RecordMergeMode;
+import org.apache.hudi.common.config.TypedProperties;
import org.apache.hudi.common.engine.EngineType;
import org.apache.hudi.common.engine.HoodieReaderContext;
import org.apache.hudi.common.fs.FSUtils;
@@ -34,6 +35,8 @@ import org.apache.hudi.common.model.OverwriteWithLatestMerger;
import org.apache.hudi.common.schema.HoodieSchema;
import org.apache.hudi.common.schema.HoodieSchemaField;
import org.apache.hudi.common.table.HoodieTableConfig;
+import org.apache.hudi.common.table.read.FileGroupReaderSchemaHandler;
+import org.apache.hudi.common.util.ConfigUtils;
import org.apache.hudi.common.util.HoodieRecordUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.ClosableIterator;
@@ -83,9 +86,11 @@ public class HudiTrinoReaderContext
ConnectorPageSource pageSource,
List<HiveColumnHandle> columnHandles,
PrefilledColumnValues prefilledColumnValues,
- LogFileParquetPageSourceFactory logPageSourceFactory)
+ LogFileParquetPageSourceFactory logPageSourceFactory,
+ TypedProperties props)
{
- super(storageConfiguration, tableConfig, Option.empty(),
Option.empty(), new AvroRecordContext(tableConfig,
tableConfig.getPayloadClass()));
+ super(storageConfiguration, tableConfig, Option.empty(),
Option.empty(),
+ new AvroRecordContext(tableConfig,
tableConfig.getPayloadClass(), ConfigUtils.getMergeProps(props, tableConfig)));
this.pageSource = pageSource;
this.prefilledColumnValues = prefilledColumnValues;
this.columnHandles = columnHandles;
@@ -100,6 +105,14 @@ public class HudiTrinoReaderContext
this.baseProjectionNames =
ImmutableSet.copyOf(colNameToHandle.keySet());
}
+ @Override
+ public void setSchemaHandler(FileGroupReaderSchemaHandler<IndexedRecord>
schemaHandler)
+ {
+ super.setSchemaHandler(schemaHandler);
+ List<String> orderingFieldNames =
HoodieRecordUtils.getOrderingFieldNames(getMergeMode(), tableConfig);
+ ((AvroRecordContext)
recordContext).initOrderingValueConverter(schemaHandler.getTableSchema(),
orderingFieldNames);
+ }
+
@Override
public ClosableIterator<IndexedRecord> getFileRecordIterator(
StoragePath storagePath,
diff --git
a/hudi-trino/src/test/java/io/trino/plugin/hudi/reader/TestHudiTrinoReaderContext.java
b/hudi-trino/src/test/java/io/trino/plugin/hudi/reader/TestHudiTrinoReaderContext.java
new file mode 100644
index 000000000000..7bf3ca3763a5
--- /dev/null
+++
b/hudi-trino/src/test/java/io/trino/plugin/hudi/reader/TestHudiTrinoReaderContext.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi.reader;
+
+import io.trino.plugin.hudi.storage.TrinoStorageConfiguration;
+import io.trino.spi.connector.EmptyPageSource;
+import org.apache.avro.LogicalTypes;
+import org.apache.avro.Schema;
+import org.apache.avro.SchemaBuilder;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.avro.generic.IndexedRecord;
+import org.apache.hudi.common.config.TypedProperties;
+import org.apache.hudi.common.model.DeleteRecord;
+import org.apache.hudi.common.schema.HoodieSchema;
+import org.apache.hudi.common.table.HoodieTableConfig;
+import org.apache.hudi.common.table.log.block.HoodieDeleteBlock;
+import org.apache.hudi.common.table.read.BufferedRecord;
+import org.apache.hudi.common.table.read.BufferedRecordMerger;
+import org.apache.hudi.common.table.read.BufferedRecordMergerFactory;
+import org.apache.hudi.common.table.read.BufferedRecords;
+import org.apache.hudi.common.table.read.FileGroupReaderSchemaHandler;
+import org.apache.hudi.common.table.read.TableState;
+import org.apache.hudi.common.util.ConfigUtils;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.storage.StoragePath;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+
+import java.io.IOException;
+import java.sql.Timestamp;
+import java.util.List;
+import java.util.Map;
+
+import static
org.apache.hudi.common.config.RecordMergeMode.EVENT_TIME_ORDERING;
+import static
org.apache.hudi.common.table.HoodieTableConfig.RECORD_MERGE_PROPERTY_PREFIX;
+import static
org.apache.hudi.keygen.constant.KeyGeneratorOptions.KEYGENERATOR_CONSISTENT_LOGICAL_TIMESTAMP_ENABLED;
+import static org.assertj.core.api.Assertions.assertThat;
+
+class TestHudiTrinoReaderContext
+{
+ @ParameterizedTest
+ @CsvSource({"9, false", "11, true"})
+ void testDateOrderingWithSerializedDelete(int deleteEpochDay, boolean
expectDeleted)
+ throws IOException
+ {
+ HoodieSchema schema =
schemaWithOrdering(LogicalTypes.date().addToSchema(Schema.create(Schema.Type.INT)));
+ TypedProperties props = new TypedProperties();
+ HudiTrinoReaderContext readerContext = createReaderContext(schema,
tableConfig(), props);
+ GenericRecord record = new GenericData.Record(schema.toAvroSchema());
+ record.put("id", "key");
+ record.put("ordering", 10);
+ BufferedRecord<IndexedRecord> data = BufferedRecords.fromEngineRecord(
+ record, "key", schema, readerContext.getRecordContext(),
List.of("ordering"), null);
+ BufferedRecord<IndexedRecord> delete = readDelete(readerContext,
deleteEpochDay);
+ BufferedRecordMerger<IndexedRecord> merger =
BufferedRecordMergerFactory.create(
+ readerContext, EVENT_TIME_ORDERING, false, Option.empty(),
schema, Option.empty(), props, Option.empty());
+
+ assertThat(merger.finalMerge(data, delete)).isSameAs(expectDeleted ?
delete : data);
+ }
+
+ @ParameterizedTest
+ @CsvSource({"false, false", "true, false", "false, true", "true, true"})
+ void testTimestampOrderingConfiguration(boolean consistentTimestamp,
boolean persistedProperty)
+ throws IOException
+ {
+ HoodieSchema schema =
schemaWithOrdering(LogicalTypes.timestampMicros().addToSchema(Schema.create(Schema.Type.LONG)));
+ HoodieTableConfig tableConfig = tableConfig();
+ TypedProperties props = new TypedProperties();
+ String key = KEYGENERATOR_CONSISTENT_LOGICAL_TIMESTAMP_ENABLED.key();
+ if (persistedProperty) {
+ tableConfig.setValue(RECORD_MERGE_PROPERTY_PREFIX + key,
Boolean.toString(consistentTimestamp));
+ // Persisted merge properties take precedence over the reader's
value.
+ props.setProperty(key, Boolean.toString(!consistentTimestamp));
+ }
+ else {
+ props.setProperty(key, Boolean.toString(consistentTimestamp));
+ }
+ HudiTrinoReaderContext readerContext = createReaderContext(schema,
tableConfig, props);
+ GenericRecord record = new GenericData.Record(schema.toAvroSchema());
+ record.put("id", "key");
+ record.put("ordering", 1_234_567L);
+ Object expected = consistentTimestamp ? new Timestamp(1234) :
1_234_567L;
+
+ assertThat(readerContext.getRecordContext().getOrderingValue(record,
schema, List.of("ordering"))).isEqualTo(expected);
+ assertThat(readDelete(readerContext,
1_234_567L).getOrderingValue()).isEqualTo(expected);
+ }
+
+ private static HoodieSchema schemaWithOrdering(Schema orderingSchema)
+ {
+ return
HoodieSchema.fromAvroSchema(SchemaBuilder.record("test_record").fields()
+ .requiredString("id")
+ .name("ordering").type(orderingSchema).noDefault()
+ .endRecord());
+ }
+
+ private static HoodieTableConfig tableConfig()
+ {
+ HoodieTableConfig tableConfig = new HoodieTableConfig();
+ tableConfig.setValue(HoodieTableConfig.VERSION, "9");
+ tableConfig.setValue(HoodieTableConfig.RECORD_MERGE_MODE,
EVENT_TIME_ORDERING.name());
+ tableConfig.setValue(HoodieTableConfig.ORDERING_FIELDS, "ordering");
+ return tableConfig;
+ }
+
+ private static HudiTrinoReaderContext createReaderContext(HoodieSchema
schema, HoodieTableConfig tableConfig, TypedProperties props)
+ {
+ // These tests consume classic log records directly; no page source is
read.
+ HudiTrinoReaderContext readerContext = new HudiTrinoReaderContext(
+ new TrinoStorageConfiguration(), tableConfig, new
EmptyPageSource(), List.of(), null,
+ (path, start, length, projection) -> {
+ throw new AssertionError("Unexpected parquet read");
+ }, props);
+ TypedProperties mergeProps = ConfigUtils.getMergeProps(props,
tableConfig);
+ readerContext.initRecordMerger(mergeProps);
+ readerContext.setHasLogFiles(false);
+ readerContext.setHasBootstrapBaseFile(false);
+ HoodieSchema projectedSchema =
HoodieSchema.fromAvroSchema(SchemaBuilder.record("test_record").fields().requiredString("id").endRecord());
+ // The ordering field can be absent from the projection, so
initialization must use the table schema.
+ readerContext.setSchemaHandler(new FileGroupReaderSchemaHandler<>(
+ readerContext, schema, projectedSchema, Option.empty(),
mergeProps,
+ TableState.of(new StoragePath("/test-table"), tableConfig,
Option.empty(), Option.empty())));
+ return readerContext;
+ }
+
+ private static BufferedRecord<IndexedRecord>
readDelete(HudiTrinoReaderContext readerContext, Comparable<?> orderingValue)
+ throws IOException
+ {
+ HoodieDeleteBlock written = new
HoodieDeleteBlock(List.of(Pair.of(DeleteRecord.create("key", "",
orderingValue), -1L)), Map.of());
+ HoodieDeleteBlock read = new
HoodieDeleteBlock(Option.of(written.getContentBytes(null).toByteArray()),
+ null, false, Option.empty(), Map.of(), Map.of());
+ return read.getRecordsToDelete(readerContext).getFirst();
+ }
+}