This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/master/pr-20544-c3526970d562365475cdbfc44664a846b7f27549 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit c32b64e98a3a3b637f793a77c83f849bf3655c57 Author: Shuo Cheng <[email protected]> AuthorDate: Thu Nov 12 11:56:30 2026 +0000 fix(common): normalize Avro ordering values for record merging (#20544) * fix(common): normalize Avro ordering values for record merging * fix(common): normalize Avro delete ordering values using field schemas * update * fix(ci): preserve locally built Hudi artifacts when switching JDKs * fix comment * fix comment * refactor(common): return normalized delete timestamps directly * refactor(common): prebuild delete ordering field converters --- .github/workflows/hudi_trino_ci.yml | 2 +- .github/workflows/hudi_trino_e2e.yml | 2 +- .../common/avro/AvroOrderingValueConverter.java | 136 ++++++++++++ .../apache/hudi/common/avro/AvroRecordContext.java | 52 ++++- .../hudi/common/avro/HoodieAvroReaderContext.java | 11 +- .../apache/hudi/common/avro/HoodieAvroUtils.java | 10 +- .../hudi/common/model/HoodieAvroIndexedRecord.java | 2 +- .../table/read/BufferedRecordMergerFactory.java | 2 +- .../hudi/common/avro/TestAvroRecordContext.java | 233 ++++++++++++++++++++- .../common/avro/TestHoodieAvroReaderContext.java | 25 +++ .../TestBufferedRecordSerializer.java | 5 +- .../read/TestBufferedRecordMergerFactory.java | 25 +++ .../buffer/TestFileGroupRecordBufferLoader.java | 1 + .../buffer/TestKeyBasedFileGroupRecordBuffer.java | 83 ++++++++ .../TestSortedKeyBasedFileGroupRecordBuffer.java | 1 + ...TestStreamingKeyBasedFileGroupRecordBuffer.java | 4 + .../TestHoodieBackedTableMetadataDataCleanup.java | 1 + .../hudi/functional/TestBufferedRecordMerger.java | 1 + .../others/TestMergeModeEventTimeOrdering.scala | 31 +++ .../sql/hudi/dml/others/TestUpdateTable.scala | 47 +++++ .../trino/plugin/hudi/HudiPageSourceProvider.java | 3 +- .../plugin/hudi/reader/HudiTrinoReaderContext.java | 17 +- .../hudi/reader/TestHudiTrinoReaderContext.java | 146 +++++++++++++ 23 files changed, 821 insertions(+), 19 deletions(-) diff --git a/.github/workflows/hudi_trino_ci.yml b/.github/workflows/hudi_trino_ci.yml index 8d60f8c30810..35cb5b5f26af 100644 --- a/.github/workflows/hudi_trino_ci.yml +++ b/.github/workflows/hudi_trino_ci.yml @@ -93,11 +93,11 @@ jobs: # they live behind the hudi-trino-tests profile but the test step needs them. run: mvn $MVN_ARGS install -pl :hudi-common,:hudi-hive-sync,:hudi-io,:hudi-sync-common,:hudi-client-common,:hudi-java-client -am -Dmaven.test.skip=true -Drat.skip -Dcheckstyle.skip - name: Set up JDK 25 + # Restoring the Maven cache again would overwrite the Hudi artifacts installed above. uses: actions/setup-java@v5 with: java-version: '25' distribution: 'temurin' - cache: maven - name: Purge Trino artifacts from the local m2 # Artifacts an older pin left behind carry the same SNAPSHOT coordinates as the current ones. run: rm -rf ~/.m2/repository/io/trino diff --git a/.github/workflows/hudi_trino_e2e.yml b/.github/workflows/hudi_trino_e2e.yml index e1a5d94c594a..9ec8fd393d09 100644 --- a/.github/workflows/hudi_trino_e2e.yml +++ b/.github/workflows/hudi_trino_e2e.yml @@ -114,11 +114,11 @@ jobs: run: mvn clean install -T 2 $SCALA_PROFILE -Dspark4.0 -Dflink1.20 -Pintegration-tests -DskipTests=true -Ddocker.compose.skip=true $MVN_ARGS - name: Set up JDK 25 + # Restoring the Maven cache again would overwrite the Hudi artifacts installed above. uses: actions/setup-java@v5 with: java-version: '25' distribution: 'temurin' - cache: maven - name: Purge Trino artifacts from the local m2 # Artifacts an older pin left behind carry the same SNAPSHOT coordinates as the current ones. run: rm -rf ~/.m2/repository/io/trino diff --git a/hudi-common/src/main/java/org/apache/hudi/common/avro/AvroOrderingValueConverter.java b/hudi-common/src/main/java/org/apache/hudi/common/avro/AvroOrderingValueConverter.java new file mode 100644 index 000000000000..89c7a03c274f --- /dev/null +++ b/hudi-common/src/main/java/org/apache/hudi/common/avro/AvroOrderingValueConverter.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.common.avro; + +import org.apache.hudi.common.schema.HoodieSchema; +import org.apache.hudi.common.schema.HoodieSchemaField; +import org.apache.hudi.common.schema.HoodieSchemaUtils; +import org.apache.hudi.common.util.collection.ArrayComparable; +import org.apache.hudi.common.util.collection.Pair; + +import org.apache.avro.Conversions.DecimalConversion; +import org.apache.avro.LogicalType; +import org.apache.avro.LogicalTypes; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericFixed; +import org.apache.avro.util.Utf8; + +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.sql.Date; +import java.sql.Timestamp; +import java.time.LocalDate; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static org.apache.hudi.common.avro.HoodieAvroUtils.extractEpochDay; +import static org.apache.hudi.common.avro.HoodieAvroUtils.extractEpochMicros; +import static org.apache.hudi.common.avro.HoodieAvroUtils.extractEpochMillis; +import static org.apache.hudi.common.avro.HoodieAvroUtils.extractLocalEpochMicros; +import static org.apache.hudi.common.avro.HoodieAvroUtils.extractLocalEpochMillis; + +/** + * Prebuilds a converter for each delete ordering field to match the comparison representation of + * {@link HoodieAvroUtils#convertValueForAvroLogicalTypes(Schema, Object, boolean)}. + * Preserves null field values and converts strings to Avro's comparable representation. + */ +public class AvroOrderingValueConverter { + private final List<Function<Comparable, Comparable>> converters; + + private AvroOrderingValueConverter(HoodieSchema dataSchema, List<String> orderingFieldNames, boolean consistentLogicalTimestampEnabled) { + this.converters = orderingFieldNames.stream() + .map(field -> HoodieSchemaUtils.getNestedField(dataSchema, field) + .map(Pair::getRight).map(HoodieSchemaField::getNonNullSchema) + .orElseThrow(() -> new IllegalArgumentException("Ordering field '" + field + "' not found in table schema"))) + .map(schema -> createConverter(schema.toAvroSchema(), consistentLogicalTimestampEnabled)) + .map(converter -> (Function<Comparable, Comparable>) value -> value == null ? null : converter.apply(value)) + .collect(Collectors.toList()); + } + + public static AvroOrderingValueConverter create(HoodieSchema dataSchema, List<String> orderingFieldNames, boolean consistentLogicalTimestampEnabled) { + return new AvroOrderingValueConverter(dataSchema, orderingFieldNames, consistentLogicalTimestampEnabled); + } + + public Comparable convert(Comparable value) { + if (converters.isEmpty()) { + return value; + } + return value instanceof ArrayComparable + ? ((ArrayComparable) value).apply(converters) + : converters.get(0).apply(value); + } + + private static Function<Comparable, Comparable> createConverter(Schema schema, boolean consistentLogicalTimestampEnabled) { + LogicalType logicalType = schema.getLogicalType(); + if (logicalType == LogicalTypes.date()) { + // Spark/Flink supply epoch days; Avro supplies LocalDate. Callers may supply java.sql.Date. + return value -> value instanceof Date ? ((Date) value).toLocalDate() : LocalDate.ofEpochDay(extractEpochDay(value)); + } + if (logicalType == LogicalTypes.timestampMillis()) { + return consistentLogicalTimestampEnabled + ? value -> new Timestamp(toEpochMillis(value)) : AvroOrderingValueConverter::toEpochMillis; + } + if (logicalType == LogicalTypes.timestampMicros()) { + // Match the shared normalizer's millisecond truncation, including negative timestamps. + return consistentLogicalTimestampEnabled + ? value -> new Timestamp(toEpochMicros(value) / 1000) : AvroOrderingValueConverter::toEpochMicros; + } + // A caller-supplied Timestamp represents wall-clock fields for a local timestamp schema. + if (logicalType == LogicalTypes.localTimestampMillis()) { + return value -> extractLocalEpochMillis(value instanceof Timestamp ? ((Timestamp) value).toLocalDateTime() : value); + } + if (logicalType == LogicalTypes.localTimestampMicros()) { + return value -> extractLocalEpochMicros(value instanceof Timestamp ? ((Timestamp) value).toLocalDateTime() : value); + } + if (logicalType instanceof LogicalTypes.Decimal) { + // Spark/Flink and Avro extraction produce BigDecimal, preserved by DecimalWrapper decoding. + DecimalConversion decimalConversion = new DecimalConversion(); + if (schema.getType() == Schema.Type.FIXED) { + return value -> value instanceof BigDecimal ? value + : decimalConversion.fromFixed((GenericFixed) value, schema, logicalType); + } else if (schema.getType() == Schema.Type.BYTES) { + return value -> { + if (value instanceof BigDecimal) { + return value; + } + ByteBuffer buffer = (ByteBuffer) value; + BigDecimal convertedValue = decimalConversion.fromBytes(buffer, schema, logicalType); + buffer.rewind(); + return convertedValue; + }; + } + } + if (schema.getType() == Schema.Type.STRING) { + // Spark/Flink Java value extraction and StringWrapper decoding produce String. + return value -> value instanceof String ? new Utf8((String) value) : value; + } + return value -> value; + } + + private static long toEpochMillis(Comparable value) { + // Avro extraction with consistent logical timestamps enabled and wrapper decoding yield Timestamp. + return value instanceof Timestamp ? ((Timestamp) value).getTime() : extractEpochMillis(value); + } + + private static long toEpochMicros(Comparable value) { + return extractEpochMicros(value instanceof Timestamp ? ((Timestamp) value).toInstant() : value); + } +} diff --git a/hudi-common/src/main/java/org/apache/hudi/common/avro/AvroRecordContext.java b/hudi-common/src/main/java/org/apache/hudi/common/avro/AvroRecordContext.java index 8f4a5fa2ddf8..d07dea1ce1a9 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/avro/AvroRecordContext.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/avro/AvroRecordContext.java @@ -19,6 +19,7 @@ package org.apache.hudi.common.avro; +import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.engine.RecordContext; import org.apache.hudi.common.model.HoodieAvroIndexedRecord; import org.apache.hudi.common.model.HoodieEmptyRecord; @@ -30,8 +31,10 @@ import org.apache.hudi.common.schema.HoodieSchemaField; import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.common.table.read.BufferedRecord; import org.apache.hudi.common.util.AvroJavaTypeConverter; +import org.apache.hudi.common.util.ConfigUtils; import org.apache.hudi.common.util.HoodieRecordUtils; import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.OrderingValues; import org.apache.hudi.exception.HoodieException; import org.apache.avro.generic.GenericData; @@ -40,6 +43,7 @@ import org.apache.avro.generic.IndexedRecord; import org.apache.avro.util.Utf8; import java.io.IOException; +import java.util.List; import java.util.Map; import java.util.Properties; import java.util.function.UnaryOperator; @@ -57,15 +61,19 @@ public class AvroRecordContext extends RecordContext<IndexedRecord> { } private final String payloadClass; + private final boolean consistentLogicalTimestampEnabled; + private AvroOrderingValueConverter orderingValueConverter; - public AvroRecordContext(HoodieTableConfig tableConfig, String payloadClass) { + public AvroRecordContext(HoodieTableConfig tableConfig, String payloadClass, TypedProperties props) { super(tableConfig, new AvroJavaTypeConverter()); this.payloadClass = payloadClass; + this.consistentLogicalTimestampEnabled = ConfigUtils.shouldKeepConsistentLogicalTimestamp(props); } public AvroRecordContext() { super(new AvroJavaTypeConverter()); this.payloadClass = null; + this.consistentLogicalTimestampEnabled = false; } public static Object getFieldValueFromIndexedRecord( @@ -107,6 +115,36 @@ public class AvroRecordContext extends RecordContext<IndexedRecord> { return getFieldValueFromIndexedRecord(record, fieldName); } + @Override + public Comparable getOrderingValue(IndexedRecord record, HoodieSchema schema, List<String> orderingFieldNames) { + return OrderingValues.create(orderingFieldNames, field -> { + Object value = getOrderingValueFromIndexedRecord(record, field, consistentLogicalTimestampEnabled); + return value == null ? OrderingValues.getDefault() : ensureComparability(value); + }); + } + + @Override + public Comparable getOrderingValue(IndexedRecord record, HoodieSchema schema, String[] orderingFieldNames) { + return orderingFieldNames == null + ? OrderingValues.getDefault() + : OrderingValues.create(orderingFieldNames, field -> { + Object value = getOrderingValueFromIndexedRecord(record, field, consistentLogicalTimestampEnabled); + return value == null ? OrderingValues.getDefault() : ensureComparability(value); + }); + } + + /** + * Extracts an ordering field from a generic Avro record, normalizing its logical type. + * + * @param record the record to read; must be a {@link GenericRecord} + * @param fieldName the field name, which may be a nested path + * @param consistentLogicalTimestampEnabled whether to use consistent logical timestamp conversion + * @return the ordering value, or null if the field or its value is missing + */ + public static Object getOrderingValueFromIndexedRecord(IndexedRecord record, String fieldName, boolean consistentLogicalTimestampEnabled) { + return HoodieAvroUtils.getNestedFieldVal((GenericRecord) record, fieldName, true, consistentLogicalTimestampEnabled); + } + @Override public String getMetaFieldValue(IndexedRecord record, int pos) { return record.get(pos).toString(); @@ -203,6 +241,18 @@ public class AvroRecordContext extends RecordContext<IndexedRecord> { return record -> HoodieAvroUtils.rewriteRecordWithNewSchema(record, to.toAvroSchema(), renamedColumns); } + /** + * Initializes schema-aware conversion so delete values use the same logical types as data records. + */ + public void initOrderingValueConverter(HoodieSchema dataSchema, List<String> orderingFieldNames) { + orderingValueConverter = AvroOrderingValueConverter.create(dataSchema, orderingFieldNames, consistentLogicalTimestampEnabled); + } + + @Override + public Comparable convertOrderingValueToEngineType(Comparable value) { + return orderingValueConverter.convert(value); + } + @Override public Comparable convertValueToEngineType(Comparable value) { if (value instanceof String) { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/avro/HoodieAvroReaderContext.java b/hudi-common/src/main/java/org/apache/hudi/common/avro/HoodieAvroReaderContext.java index 788810b7ba3f..be60facac9b7 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/avro/HoodieAvroReaderContext.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/avro/HoodieAvroReaderContext.java @@ -39,6 +39,7 @@ import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.common.table.log.InstantRange; import org.apache.hudi.common.table.read.BufferedRecord; import org.apache.hudi.common.table.read.BufferedRecordSerializer; +import org.apache.hudi.common.table.read.FileGroupReaderSchemaHandler; import org.apache.hudi.common.util.ConfigUtils; import org.apache.hudi.common.util.HoodieRecordUtils; import org.apache.hudi.common.util.Option; @@ -153,11 +154,19 @@ public class HoodieAvroReaderContext extends HoodieReaderContext<IndexedRecord> Map<StoragePath, HoodieAvroFileReader> reusableFileReaders, String payloadClassName, HoodieConfig hoodieReaderConfig) { - super(storageConfiguration, tableConfig, instantRangeOpt, filterOpt, new AvroRecordContext(tableConfig, payloadClassName), hoodieReaderConfig); + super(storageConfiguration, tableConfig, instantRangeOpt, filterOpt, + new AvroRecordContext(tableConfig, payloadClassName, ConfigUtils.getMergeProps(hoodieReaderConfig.getProps(), tableConfig)), hoodieReaderConfig); this.reusableFileReaders = reusableFileReaders; this.isMultiFormat = tableConfig.isMultipleBaseFileFormatsEnabled(); } + @Override + public void setSchemaHandler(FileGroupReaderSchemaHandler<IndexedRecord> schemaHandler) { + super.setSchemaHandler(schemaHandler); + List<String> orderingFieldNames = HoodieRecordUtils.getOrderingFieldNames(getMergeMode(), tableConfig); + ((AvroRecordContext) recordContext).initOrderingValueConverter(schemaHandler.getTableSchema(), orderingFieldNames); + } + @Override public ClosableIterator<IndexedRecord> getFileRecordIterator( StoragePathInfo storagePathInfo, long start, long length, HoodieSchema dataSchema, HoodieSchema requiredSchema, diff --git a/hudi-common/src/main/java/org/apache/hudi/common/avro/HoodieAvroUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/avro/HoodieAvroUtils.java index 21485fe7876c..5f768ab203d2 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/avro/HoodieAvroUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/avro/HoodieAvroUtils.java @@ -906,7 +906,7 @@ public class HoodieAvroUtils { // fastReaderEnabled=true), and return the canonical primitive for the same underlying bytes. // See the javadoc on convertValueForAvroLogicalTypes for context. - private static long extractEpochDay(Object fieldValue) { + static long extractEpochDay(Object fieldValue) { if (fieldValue instanceof LocalDate) { return ((LocalDate) fieldValue).toEpochDay(); } @@ -916,7 +916,7 @@ public class HoodieAvroUtils { return Long.parseLong(fieldValue.toString()); } - private static long extractEpochMillis(Object fieldValue) { + static long extractEpochMillis(Object fieldValue) { if (fieldValue instanceof Instant) { return ((Instant) fieldValue).toEpochMilli(); } @@ -926,7 +926,7 @@ public class HoodieAvroUtils { return Long.parseLong(fieldValue.toString()); } - private static long extractEpochMicros(Object fieldValue) { + static long extractEpochMicros(Object fieldValue) { if (fieldValue instanceof Instant) { Instant instant = (Instant) fieldValue; return Math.addExact(Math.multiplyExact(instant.getEpochSecond(), 1_000_000L), instant.getNano() / 1000L); @@ -937,7 +937,7 @@ public class HoodieAvroUtils { return Long.parseLong(fieldValue.toString()); } - private static long extractLocalEpochMillis(Object fieldValue) { + static long extractLocalEpochMillis(Object fieldValue) { if (fieldValue instanceof LocalDateTime) { return ((LocalDateTime) fieldValue).toInstant(ZoneOffset.UTC).toEpochMilli(); } @@ -947,7 +947,7 @@ public class HoodieAvroUtils { return Long.parseLong(fieldValue.toString()); } - private static long extractLocalEpochMicros(Object fieldValue) { + static long extractLocalEpochMicros(Object fieldValue) { if (fieldValue instanceof LocalDateTime) { Instant instant = ((LocalDateTime) fieldValue).toInstant(ZoneOffset.UTC); return Math.addExact(Math.multiplyExact(instant.getEpochSecond(), 1_000_000L), instant.getNano() / 1000L); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieAvroIndexedRecord.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieAvroIndexedRecord.java index eda401c5b8e5..c17f4b824ff2 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieAvroIndexedRecord.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieAvroIndexedRecord.java @@ -315,7 +315,7 @@ public class HoodieAvroIndexedRecord extends HoodieRecord<IndexedRecord> { KeyGeneratorOptions.KEYGENERATOR_CONSISTENT_LOGICAL_TIMESTAMP_ENABLED.defaultValue())); return OrderingValues.create( orderingFields, - field -> (Comparable<?>) HoodieAvroUtils.getNestedFieldVal((GenericRecord) data, field, true, consistentLogicalTimestampEnabled)); + field -> (Comparable<?>) AvroRecordContext.getOrderingValueFromIndexedRecord(data, field, consistentLogicalTimestampEnabled)); } private void decodeRecord(Schema recordSchema) { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/read/BufferedRecordMergerFactory.java b/hudi-common/src/main/java/org/apache/hudi/common/table/read/BufferedRecordMergerFactory.java index 4484abaefee9..45b0d2801086 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/read/BufferedRecordMergerFactory.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/read/BufferedRecordMergerFactory.java @@ -493,7 +493,7 @@ public class BufferedRecordMergerFactory { return Option.empty(); } Comparable existingOrderingVal = existingRecord.getOrderingValue(); - Comparable deleteOrderingVal = recordContext.convertOrderingValueToEngineType(deleteRecord.getOrderingValue()); + Comparable deleteOrderingVal = recordContext.getOrderingValue(deleteRecord); // Checks the ordering value does not equal to 0 // because we use 0 as the default value which means natural order boolean chooseExisting = !OrderingValues.isDefault(deleteOrderingVal) diff --git a/hudi-common/src/test/java/org/apache/hudi/common/avro/TestAvroRecordContext.java b/hudi-common/src/test/java/org/apache/hudi/common/avro/TestAvroRecordContext.java index 6972b93fc735..b35fdb74357e 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/avro/TestAvroRecordContext.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/avro/TestAvroRecordContext.java @@ -19,10 +19,21 @@ package org.apache.hudi.common.avro; +import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.common.model.DeleteRecord; +import org.apache.hudi.common.model.HoodieAvroIndexedRecord; import org.apache.hudi.common.model.MetaFieldsMode; import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.table.HoodieTableConfig; +import org.apache.hudi.common.table.log.block.HoodieDeleteBlock; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.OrderingValues; +import org.apache.hudi.common.util.collection.ArrayComparable; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.keygen.constant.KeyGeneratorOptions; +import org.apache.avro.Conversions; +import org.apache.avro.LogicalTypes; import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecord; @@ -31,9 +42,22 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; +import java.io.IOException; +import java.math.BigDecimal; +import java.sql.Date; +import java.sql.Timestamp; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.stream.Stream; @@ -61,6 +85,213 @@ class TestAvroRecordContext { + "{\"name\":\"rec_map\",\"type\":[\"null\",{\"type\":\"map\",\"values\":{\"type\":\"record\"," + "\"name\":\"inner\",\"fields\":[{\"name\":\"x\",\"type\":\"int\"}]}}],\"default\":null}]}"); + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testOrderingValuesMatchIncomingRecord(boolean consistentTimestamp) { + Schema schema = new Schema.Parser().parse("{\"type\":\"record\",\"name\":\"ordering\",\"fields\":[" + + "{\"name\":\"ts\",\"type\":{\"type\":\"long\",\"logicalType\":\"timestamp-micros\"}}," + + "{\"name\":\"date\",\"type\":{\"type\":\"int\",\"logicalType\":\"date\"}}," + + "{\"name\":\"decimal\",\"type\":{\"type\":\"fixed\",\"name\":\"fixed_decimal\",\"size\":5," + + "\"logicalType\":\"decimal\",\"precision\":10,\"scale\":2}}," + + "{\"name\":\"nested\",\"type\":{\"type\":\"record\",\"name\":\"inner_date\",\"fields\":[" + + "{\"name\":\"date\",\"type\":{\"type\":\"int\",\"logicalType\":\"date\"}}]}}]}"); + GenericRecord record = new GenericData.Record(schema); + record.put("ts", 1790812800000000L); + record.put("date", 20727); + Schema decimalSchema = schema.getField("decimal").schema(); + record.put("decimal", new Conversions.DecimalConversion().toFixed(new BigDecimal("1.50"), decimalSchema, LogicalTypes.decimal(10, 2))); + GenericRecord nested = new GenericData.Record(schema.getField("nested").schema()); + nested.put("date", 20727); + record.put("nested", nested); + HoodieSchema hoodieSchema = HoodieSchema.fromAvroSchema(schema); + TypedProperties props = new TypedProperties(); + props.setProperty(KeyGeneratorOptions.KEYGENERATOR_CONSISTENT_LOGICAL_TIMESTAMP_ENABLED.key(), String.valueOf(consistentTimestamp)); + AvroRecordContext context = new AvroRecordContext(new HoodieTableConfig(), null, props); + + for (String field : Arrays.asList("ts", "date", "decimal", "nested.date")) { + Comparable incoming = new HoodieAvroIndexedRecord(record).getOrderingValue(hoodieSchema, props, new String[] {field}); + assertEquals(incoming, context.getOrderingValue(record, hoodieSchema, new String[] {field}), field); + assertEquals(incoming, context.getOrderingValue(record, hoodieSchema, Arrays.asList(field)), field); + } + assertEquals(OrderingValues.getDefault(), context.getOrderingValue(record, hoodieSchema, new String[] {"missing"})); + assertNull(new HoodieAvroIndexedRecord(record).getOrderingValue(hoodieSchema, props, new String[] {"missing"})); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testDeleteOrderingValuesMatchData(boolean consistentTimestamp) throws IOException { + Schema schema = new Schema.Parser().parse("{\"type\":\"record\",\"name\":\"delete_ordering\",\"fields\":[" + + "{\"name\":\"date\",\"type\":[\"null\",{\"type\":\"int\",\"logicalType\":\"date\"}],\"default\":null}," + + "{\"name\":\"ts_millis\",\"type\":{\"type\":\"long\",\"logicalType\":\"timestamp-millis\"}}," + + "{\"name\":\"ts_micros\",\"type\":{\"type\":\"long\",\"logicalType\":\"timestamp-micros\"}}," + + "{\"name\":\"decimal_fixed\",\"type\":{\"type\":\"fixed\",\"name\":\"decimal\",\"size\":5," + + "\"logicalType\":\"decimal\",\"precision\":10,\"scale\":2}}," + + "{\"name\":\"decimal_bytes\",\"type\":{\"type\":\"bytes\",\"logicalType\":\"decimal\",\"precision\":10,\"scale\":2}}," + + "{\"name\":\"plain_int\",\"type\":\"int\"},{\"name\":\"plain_long\",\"type\":\"long\"}," + + "{\"name\":\"text\",\"type\":\"string\"}," + + "{\"name\":\"nested\",\"type\":{\"type\":\"record\",\"name\":\"nested_ordering\",\"fields\":[" + + "{\"name\":\"date\",\"type\":{\"type\":\"int\",\"logicalType\":\"date\"}}]}}]}"); + Map<String, Comparable> deleteValues = new LinkedHashMap<>(); + deleteValues.put("date", 20727); + deleteValues.put("ts_millis", 1790812800123L); + deleteValues.put("ts_micros", 1790812800123456L); + deleteValues.put("decimal_fixed", new BigDecimal("1.50")); + deleteValues.put("decimal_bytes", new BigDecimal("2.50")); + deleteValues.put("plain_int", 20727); + deleteValues.put("plain_long", 1790812800123456L); + deleteValues.put("text", "value"); + GenericRecord record = new GenericData.Record(schema); + deleteValues.forEach(record::put); + record.put("text", new Utf8("value")); + Schema fixedSchema = schema.getField("decimal_fixed").schema(); + Schema bytesSchema = schema.getField("decimal_bytes").schema(); + record.put("decimal_fixed", new Conversions.DecimalConversion().toFixed( + (BigDecimal) deleteValues.get("decimal_fixed"), fixedSchema, fixedSchema.getLogicalType())); + record.put("decimal_bytes", new Conversions.DecimalConversion().toBytes( + (BigDecimal) deleteValues.get("decimal_bytes"), bytesSchema, bytesSchema.getLogicalType())); + GenericRecord nested = new GenericData.Record(schema.getField("nested").schema()); + nested.put("date", 20727); + record.put("nested", nested); + deleteValues.put("nested.date", 20727); + HoodieSchema hoodieSchema = HoodieSchema.fromAvroSchema(schema); + TypedProperties props = new TypedProperties(); + props.setProperty(KeyGeneratorOptions.KEYGENERATOR_CONSISTENT_LOGICAL_TIMESTAMP_ENABLED.key(), String.valueOf(consistentTimestamp)); + AvroRecordContext context = new AvroRecordContext(new HoodieTableConfig(), null, props); + + for (Map.Entry<String, Comparable> entry : deleteValues.entrySet()) { + List<String> fields = Collections.singletonList(entry.getKey()); + context.initOrderingValueConverter(hoodieSchema, fields); + Comparable dataOrdering = context.getOrderingValue(record, hoodieSchema, fields); + Comparable deleteOrdering = context.getOrderingValue(roundTripDelete(entry.getValue())); + assertEquals(0, dataOrdering.compareTo(deleteOrdering), entry.getKey()); + assertNull(context.convertOrderingValueToEngineType(null), entry.getKey()); + if (entry.getValue() instanceof BigDecimal) { + assertEquals(dataOrdering, context.convertOrderingValueToEngineType((Comparable) record.get(entry.getKey()))); + } + } + + // SQL date/timestamp wrappers decode to different Java representations than primitive wrappers. + context.initOrderingValueConverter(hoodieSchema, Collections.singletonList("date")); + LocalDate date = LocalDate.ofEpochDay(20727); + assertEquals(date, context.getOrderingValue(roundTripDelete(Date.valueOf(date)))); + assertEquals(date, context.convertOrderingValueToEngineType(date)); + assertEquals(date, context.convertOrderingValueToEngineType("20727")); + assertEquals(date, context.convertOrderingValueToEngineType(new Utf8("20727"))); + assertThrows(NumberFormatException.class, () -> context.convertOrderingValueToEngineType("invalid")); + for (String field : Arrays.asList("ts_millis", "ts_micros")) { + List<String> fields = Collections.singletonList(field); + context.initOrderingValueConverter(hoodieSchema, fields); + long rawValue = (long) deleteValues.get(field); + Instant instant = field.equals("ts_millis") ? Instant.ofEpochMilli(rawValue) + : Instant.ofEpochSecond(rawValue / 1_000_000, (rawValue % 1_000_000) * 1_000); + Comparable dataOrdering = context.getOrderingValue(record, hoodieSchema, fields); + assertEquals(0, dataOrdering.compareTo(context.getOrderingValue(roundTripDelete(Timestamp.from(instant))))); + } + + List<String> fields = new ArrayList<>(deleteValues.keySet()); + context.initOrderingValueConverter(hoodieSchema, fields); + Comparable multipleValues = OrderingValues.create(deleteValues.values().toArray(new Comparable[0])); + Comparable decodedValues = roundTripDelete(multipleValues).getOrderingValue(); + Comparable dataOrdering = context.getOrderingValue(record, hoodieSchema, fields); + assertEquals(0, dataOrdering.compareTo(context.convertOrderingValueToEngineType(decodedValues))); + } + + @ParameterizedTest + @ValueSource(strings = {"local-timestamp-millis", "local-timestamp-micros"}) + void testLocalTimestampDeleteOrdering(String logicalType) throws IOException { + Schema schema = new Schema.Parser().parse("{\"type\":\"record\",\"name\":\"local_timestamp\",\"fields\":[" + + "{\"name\":\"ts\",\"type\":{\"type\":\"long\",\"logicalType\":\"" + logicalType + "\"}}]}"); + HoodieSchema hoodieSchema = HoodieSchema.fromAvroSchema(schema); + Instant instant = Instant.parse("2026-10-01T00:00:00.123456Z"); + LocalDateTime localTime = LocalDateTime.ofInstant(instant, ZoneOffset.UTC); + long rawValue = logicalType.equals("local-timestamp-millis") ? instant.toEpochMilli() + : instant.getEpochSecond() * 1_000_000 + instant.getNano() / 1_000; + GenericRecord record = new GenericData.Record(schema); + record.put("ts", rawValue); + TypedProperties props = new TypedProperties(); + props.setProperty(KeyGeneratorOptions.KEYGENERATOR_CONSISTENT_LOGICAL_TIMESTAMP_ENABLED.key(), "true"); + AvroRecordContext context = new AvroRecordContext(new HoodieTableConfig(), null, props); + List<String> fields = Collections.singletonList("ts"); + context.initOrderingValueConverter(hoodieSchema, fields); + Comparable expected = context.getOrderingValue(record, hoodieSchema, fields); + assertEquals(expected, context.getOrderingValue(roundTripDelete(rawValue))); + assertEquals(expected, context.getOrderingValue(roundTripDelete(Timestamp.valueOf(localTime)))); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testTimestampDeleteOrderingPrecision(boolean consistentTimestamp) { + TypedProperties props = new TypedProperties(); + props.setProperty(KeyGeneratorOptions.KEYGENERATOR_CONSISTENT_LOGICAL_TIMESTAMP_ENABLED.key(), String.valueOf(consistentTimestamp)); + AvroRecordContext context = new AvroRecordContext(new HoodieTableConfig(), null, props); + for (String logicalType : Arrays.asList("timestamp-millis", "timestamp-micros", "local-timestamp-millis", "local-timestamp-micros")) { + Schema schema = new Schema.Parser().parse("{\"type\":\"record\",\"name\":\"timestamp_ordering\",\"fields\":[" + + "{\"name\":\"ts\",\"type\":{\"type\":\"long\",\"logicalType\":\"" + logicalType + "\"}}]}"); + context.initOrderingValueConverter(HoodieSchema.fromAvroSchema(schema), Collections.singletonList("ts")); + for (long micros : new long[] {-1001L, 1790812800123456L}) { + Instant instant = Instant.ofEpochSecond(Math.floorDiv(micros, 1_000_000L), Math.floorMod(micros, 1_000_000L) * 1000L); + long rawValue = logicalType.endsWith("millis") ? instant.toEpochMilli() : micros; + Object expected = HoodieAvroUtils.convertValueForAvroLogicalTypes(schema.getField("ts").schema(), rawValue, consistentTimestamp); + // Local timestamps preserve wall-clock fields rather than the instant in the JVM's timezone. + Timestamp timestamp = logicalType.startsWith("local-") + ? Timestamp.valueOf(LocalDateTime.ofInstant(instant, ZoneOffset.UTC)) : Timestamp.from(instant); + assertEquals(expected, context.convertOrderingValueToEngineType(timestamp), logicalType); + assertEquals(expected, context.convertOrderingValueToEngineType(rawValue), logicalType); + Comparable javaTimeValue = logicalType.startsWith("local-") ? LocalDateTime.ofInstant(instant, ZoneOffset.UTC) : instant; + assertEquals(expected, context.convertOrderingValueToEngineType(javaTimeValue), logicalType); + } + } + } + + @Test + void testMultiFieldDeleteOrderingWithEpochDate() throws IOException { + HoodieSchema schema = HoodieSchema.fromAvroSchema(new Schema.Parser().parse( + "{\"type\":\"record\",\"name\":\"epoch_date\",\"fields\":[" + + "{\"name\":\"rank\",\"type\":\"int\"}," + + "{\"name\":\"date\",\"type\":{\"type\":\"int\",\"logicalType\":\"date\"}}]}")); + GenericRecord record = new GenericData.Record(schema.toAvroSchema()); + record.put("rank", 1); + record.put("date", 0); + List<String> fields = Arrays.asList("rank", "date"); + AvroRecordContext context = new AvroRecordContext(); + context.initOrderingValueConverter(schema, fields); + + Comparable dataOrdering = context.getOrderingValue(record, schema, fields); + Comparable deleteOrdering = context.getOrderingValue(roundTripDelete(new ArrayComparable(new Comparable[] {1, 0}))); + assertEquals(0, dataOrdering.compareTo(deleteOrdering)); + } + + @Test + void testDeleteOrderingPreservesSentinelsAndValidatesFields() { + HoodieSchema schema = HoodieSchema.fromAvroSchema(new Schema.Parser().parse( + "{\"type\":\"record\",\"name\":\"sentinels\",\"fields\":[" + + "{\"name\":\"date\",\"type\":{\"type\":\"int\",\"logicalType\":\"date\"}}," + + "{\"name\":\"ts\",\"type\":{\"type\":\"long\",\"logicalType\":\"timestamp-micros\"}}]}")); + AvroRecordContext context = new AvroRecordContext(); + context.initOrderingValueConverter(schema, Arrays.asList("date", "ts")); + assertEquals(OrderingValues.getDefault(), context.getOrderingValue(DeleteRecord.create("key", "", null))); + assertEquals(OrderingValues.getDefault(), context.getOrderingValue(DeleteRecord.create("key", "", OrderingValues.getDefault()))); + assertNull(context.convertOrderingValueToEngineType(null)); + assertEquals(LocalDate.ofEpochDay(0), context.convertOrderingValueToEngineType(0)); + ArrayComparable values = new ArrayComparable(new Comparable[] {0, null}); + ArrayComparable expected = new ArrayComparable(new Comparable[] {LocalDate.ofEpochDay(0), null}); + assertEquals(expected, context.getOrderingValue(DeleteRecord.create("key", "", values))); + + IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, + () -> context.initOrderingValueConverter(schema, Collections.singletonList("missing"))); + assertEquals("Ordering field 'missing' not found in table schema", exception.getMessage()); + context.initOrderingValueConverter(schema, Collections.emptyList()); + assertEquals("value", context.getOrderingValue(DeleteRecord.create("key", "", "value"))); + } + + private static DeleteRecord roundTripDelete(Comparable value) throws IOException { + HoodieDeleteBlock written = new HoodieDeleteBlock( + Collections.singletonList(Pair.of(DeleteRecord.create("key", "", value), -1L)), Collections.emptyMap()); + HoodieDeleteBlock read = new HoodieDeleteBlock(Option.of(written.getContentBytes(null).toByteArray()), + null, false, Option.empty(), Collections.emptyMap(), Collections.emptyMap()); + return read.getRecordsToDelete()[0]; + } + private static Stream<Arguments> testConvertValueToEngineType() { return Stream.of( Arguments.of(1L, 1L), @@ -96,7 +327,7 @@ class TestAvroRecordContext { if (partitionFields != null) { tableConfig.setValue(HoodieTableConfig.PARTITION_FIELDS, partitionFields); } - AvroRecordContext context = new AvroRecordContext(tableConfig, null); + AvroRecordContext context = new AvroRecordContext(tableConfig, null, new TypedProperties()); assertEquals(expectedKey, context.getRecordKey(buildRecord(), HoodieSchema.fromAvroSchema(RECORD_SCHEMA))); } diff --git a/hudi-common/src/test/java/org/apache/hudi/common/avro/TestHoodieAvroReaderContext.java b/hudi-common/src/test/java/org/apache/hudi/common/avro/TestHoodieAvroReaderContext.java index 927158d890dd..2c59f76ea4c7 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/avro/TestHoodieAvroReaderContext.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/avro/TestHoodieAvroReaderContext.java @@ -20,10 +20,12 @@ package org.apache.hudi.common.avro; import org.apache.hudi.common.model.DefaultHoodieRecordPayload; +import org.apache.hudi.common.model.DeleteRecord; import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.schema.HoodieSchemaUtils; import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.common.table.read.BufferedRecord; +import org.apache.hudi.common.table.read.FileGroupReaderSchemaHandler; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.ClosableIterator; import org.apache.hudi.storage.StorageConfiguration; @@ -35,6 +37,7 @@ import org.apache.avro.generic.IndexedRecord; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import java.time.LocalDate; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -62,6 +65,28 @@ class TestHoodieAvroReaderContext { @BeforeEach void setup() { when(tableConfig.getPayloadClass()).thenReturn(DefaultHoodieRecordPayload.class.getName()); + when(tableConfig.getPayloadClassIfPresent()).thenReturn(Option.empty()); + } + + @Test + void initOrderingValueConverterUsesTableSchema() { + HoodieSchema fullSchema = HoodieSchema.fromAvroSchema(new Schema.Parser().parse( + "{\"type\":\"record\",\"name\":\"table_record\",\"fields\":[" + + "{\"name\":\"id\",\"type\":\"string\"}," + + "{\"name\":\"date\",\"type\":{\"type\":\"int\",\"logicalType\":\"date\"}}]}")); + HoodieSchema projectedSchema = HoodieSchema.fromAvroSchema(new Schema.Parser().parse( + "{\"type\":\"record\",\"name\":\"table_record\",\"fields\":[{\"name\":\"id\",\"type\":\"string\"}]}")); + when(tableConfig.getOrderingFields()).thenReturn(Collections.singletonList("date")); + HoodieAvroReaderContext readerContext = getReaderContextWithMetaFields(); + readerContext.setHasLogFiles(false); + FileGroupReaderSchemaHandler<IndexedRecord> schemaHandler = mock(FileGroupReaderSchemaHandler.class); + when(schemaHandler.getTableSchema()).thenReturn(fullSchema); + // Base-file-only reads may project away the ordering field. + when(schemaHandler.getRequiredSchema()).thenReturn(projectedSchema); + readerContext.setSchemaHandler(schemaHandler); + + assertEquals(LocalDate.ofEpochDay(10), + readerContext.getRecordContext().getOrderingValue(DeleteRecord.create("key", "", 10))); } @Test diff --git a/hudi-common/src/test/java/org/apache/hudi/common/serialization/TestBufferedRecordSerializer.java b/hudi-common/src/test/java/org/apache/hudi/common/serialization/TestBufferedRecordSerializer.java index 0f189be7b6d8..8dc6b4ff5150 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/serialization/TestBufferedRecordSerializer.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/serialization/TestBufferedRecordSerializer.java @@ -19,9 +19,7 @@ package org.apache.hudi.common.serialization; import org.apache.hudi.avro.model.HoodieMetadataRecord; -import org.apache.hudi.common.avro.AvroRecordContext; import org.apache.hudi.common.avro.AvroRecordSerializer; -import org.apache.hudi.common.model.DeleteRecord; import org.apache.hudi.common.model.HoodieOperation; import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.table.read.BufferedRecord; @@ -101,8 +99,7 @@ public class TestBufferedRecordSerializer { @EnumSource(value = HoodieOperation.class, names = {"UPDATE_BEFORE", "DELETE"}) void testDeleteRecordSerAndDe(HoodieOperation hoodieOperation) throws IOException { HoodieSchema schema = SchemaTestUtil.getSimpleSchema(); - DeleteRecord record = DeleteRecord.create("id", "partition", 100); - BufferedRecord<IndexedRecord> bufferedRecord = BufferedRecords.fromDeleteRecord(record, AvroRecordContext.getFieldAccessorInstance()); + BufferedRecord<IndexedRecord> bufferedRecord = BufferedRecords.createDelete("id", 100); bufferedRecord.setHoodieOperation(hoodieOperation); AvroRecordSerializer avroRecordSerializer = new AvroRecordSerializer(integer -> schema.toAvroSchema()); diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/read/TestBufferedRecordMergerFactory.java b/hudi-common/src/test/java/org/apache/hudi/common/table/read/TestBufferedRecordMergerFactory.java index 8015d0dcd8f9..2db89501d46d 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/read/TestBufferedRecordMergerFactory.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/read/TestBufferedRecordMergerFactory.java @@ -23,6 +23,7 @@ import org.apache.hudi.common.config.RecordMergeMode; import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.engine.HoodieReaderContext; import org.apache.hudi.common.engine.RecordContext; +import org.apache.hudi.common.model.DeleteRecord; import org.apache.hudi.common.model.HoodieRecordMerger; import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.schema.HoodieSchemaType; @@ -32,7 +33,12 @@ import org.apache.hudi.common.util.collection.Pair; import org.junit.jupiter.api.Test; +import java.io.IOException; +import java.time.LocalDate; + import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.CALLS_REAL_METHODS; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -68,6 +74,25 @@ class TestBufferedRecordMergerFactory { .getClass().getSimpleName()); } + @Test + void testLegacyDeleteMergePreservesSentinelsBeforeConversion() throws IOException { + RecordContext<String> recordContext = mock(RecordContext.class, CALLS_REAL_METHODS); + when(recordContext.convertOrderingValueToEngineType(any())).thenAnswer( + invocation -> LocalDate.ofEpochDay(((Number) invocation.getArgument(0)).longValue())); + HoodieReaderContext<String> readerContext = mock(HoodieReaderContext.class); + when(readerContext.getRecordContext()).thenReturn(recordContext); + BufferedRecordMerger<String> merger = BufferedRecordMergerFactory.create( + readerContext, RecordMergeMode.EVENT_TIME_ORDERING, false, Option.empty(), + HoodieSchema.create(HoodieSchemaType.STRING), Option.empty(), new TypedProperties(), Option.empty()); + BufferedRecord<String> existing = new BufferedRecord<>("key", LocalDate.ofEpochDay(10), "record", 0, null); + + for (Comparable orderingValue : new Comparable[] {null, 0}) { + DeleteRecord delete = DeleteRecord.create("key", "", orderingValue); + assertEquals(Option.of(delete), merger.deltaMerge(delete, existing)); + } + assertEquals(Option.empty(), merger.deltaMerge(DeleteRecord.create("key", "", 9), existing)); + } + private static void assertMerger(String expected, HoodieReaderContext<String> context, RecordMergeMode mode, diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestFileGroupRecordBufferLoader.java b/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestFileGroupRecordBufferLoader.java index 16a2eb893f95..f639aaecdc92 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestFileGroupRecordBufferLoader.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestFileGroupRecordBufferLoader.java @@ -63,6 +63,7 @@ public class TestFileGroupRecordBufferLoader extends BaseTestFileGroupRecordBuff : FileGroupRecordBufferLoader.createStreamingRecordsBufferLoader(); HoodieReadStats readStats = new HoodieReadStats(); HoodieTableConfig tableConfig = mock(HoodieTableConfig.class); + when(tableConfig.getPayloadClassIfPresent()).thenReturn(Option.empty()); when(tableConfig.getRecordMergeMode()).thenReturn(RecordMergeMode.COMMIT_TIME_ORDERING); when(tableConfig.getTableVersion()).thenReturn(HoodieTableVersion.NINE); when(tableConfig.getPartialUpdateMode()).thenReturn(Option.empty()); diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestKeyBasedFileGroupRecordBuffer.java b/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestKeyBasedFileGroupRecordBuffer.java index 60c0552a444a..d9a6df1fc4f5 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestKeyBasedFileGroupRecordBuffer.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestKeyBasedFileGroupRecordBuffer.java @@ -21,21 +21,32 @@ package org.apache.hudi.common.table.read.buffer; import org.apache.hudi.common.avro.HoodieAvroReaderContext; import org.apache.hudi.common.config.RecordMergeMode; +import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.engine.HoodieReaderContext; import org.apache.hudi.common.model.DeleteRecord; import org.apache.hudi.common.model.HoodieAvroRecordMerger; +import org.apache.hudi.common.schema.HoodieSchema; +import org.apache.hudi.common.schema.HoodieSchemaField; +import org.apache.hudi.common.schema.HoodieSchemaType; +import org.apache.hudi.common.schema.internal.InternalSchema; import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.common.table.log.block.HoodieDataBlock; import org.apache.hudi.common.table.log.block.HoodieDeleteBlock; import org.apache.hudi.common.table.read.BufferedRecord; +import org.apache.hudi.common.table.read.DeleteContext; +import org.apache.hudi.common.table.read.FileGroupReaderSchemaHandler; import org.apache.hudi.common.table.read.HoodieReadStats; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.ClosableIterator; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.storage.StorageConfiguration; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import java.io.IOException; import java.io.Serializable; @@ -47,6 +58,7 @@ import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -93,6 +105,7 @@ class TestKeyBasedFileGroupRecordBuffer extends BaseTestFileGroupRecordBuffer { void readWithEventTimeOrdering() throws IOException { HoodieReadStats readStats = new HoodieReadStats(); HoodieTableConfig tableConfig = mock(HoodieTableConfig.class); + when(tableConfig.getPayloadClassIfPresent()).thenReturn(Option.empty()); when(tableConfig.getRecordKeyFields()).thenReturn(Option.of(new String[] {"record_key"})); when(tableConfig.getPartitionFields()).thenReturn(Option.empty()); StorageConfiguration<?> storageConfiguration = mock(StorageConfiguration.class); @@ -121,6 +134,7 @@ class TestKeyBasedFileGroupRecordBuffer extends BaseTestFileGroupRecordBuffer { void readWithEventTimeOrderingAndDeleteBlock() throws IOException { HoodieReadStats readStats = new HoodieReadStats(); HoodieTableConfig tableConfig = mock(HoodieTableConfig.class); + when(tableConfig.getPayloadClassIfPresent()).thenReturn(Option.empty()); when(tableConfig.getRecordKeyFields()).thenReturn(Option.of(new String[] {"record_key"})); when(tableConfig.getPartitionFields()).thenReturn(Option.empty()); StorageConfiguration<?> storageConfiguration = mock(StorageConfiguration.class); @@ -152,10 +166,77 @@ class TestKeyBasedFileGroupRecordBuffer extends BaseTestFileGroupRecordBuffer { assertEquals(2, readStats.getNumUpdates()); } + @ParameterizedTest + @CsvSource({ + "9, false, false", + "9, true, false", + "10, false, true", + "10, true, false", + "11, false, true", + "11, true, true" + }) + void readWithDateOrderingAndSerializedDeleteBlock(int deleteEpochDay, boolean deleteFirst, boolean expectDeleted) throws IOException { + HoodieSchema dateSchema = HoodieSchema.createRecord("date_record", "namespace", null, + Arrays.asList( + HoodieSchemaField.of("record_key", HoodieSchema.create(HoodieSchemaType.STRING)), + HoodieSchemaField.of("ts", HoodieSchema.create(HoodieSchemaType.DATE)))); + GenericRecord record = new GenericData.Record(dateSchema.toAvroSchema()); + record.put("record_key", "1"); + record.put("ts", 10); + + TypedProperties props = new TypedProperties(); + props.setProperty(HoodieTableConfig.ORDERING_FIELDS.key(), "ts"); + HoodieTableConfig tableConfig = mock(HoodieTableConfig.class); + when(tableConfig.getPayloadClassIfPresent()).thenReturn(Option.empty()); + when(tableConfig.getRecordKeyFields()).thenReturn(Option.of(new String[] {"record_key"})); + when(tableConfig.getPartitionFields()).thenReturn(Option.empty()); + when(tableConfig.getOrderingFields()).thenReturn(Collections.singletonList("ts")); + HoodieReaderContext<IndexedRecord> readerContext = new HoodieAvroReaderContext( + mock(StorageConfiguration.class), tableConfig, Option.empty(), Option.empty(), props); + FileGroupReaderSchemaHandler<IndexedRecord> schemaHandler = mock(FileGroupReaderSchemaHandler.class); + when(schemaHandler.getTableSchema()).thenReturn(dateSchema); + when(schemaHandler.getRequiredSchema()).thenReturn(dateSchema); + when(schemaHandler.getSchemaForUpdates()).thenReturn(dateSchema); + when(schemaHandler.getInternalSchema()).thenReturn(InternalSchema.getEmptyInternalSchema()); + when(schemaHandler.getSchemaEvolutionTransformer(any(), any())).thenReturn(Option.empty()); + when(schemaHandler.getDeleteContext()).thenReturn(new DeleteContext(props, dateSchema)); + readerContext.setSchemaHandler(schemaHandler); + + HoodieDataBlock dataBlock = mock(HoodieDataBlock.class); + when(dataBlock.getSchema()).thenReturn(dateSchema); + when(dataBlock.getEngineRecordIterator(readerContext)) + .thenReturn(ClosableIterator.wrap(Collections.<IndexedRecord>singletonList(record).iterator())); + // Inline delete blocks retain the physical integer even though Avro DATE ordering is a LocalDate. + HoodieDeleteBlock writtenDeleteBlock = new HoodieDeleteBlock( + Collections.singletonList(Pair.of(DeleteRecord.create("1", "", deleteEpochDay), -1L)), Collections.emptyMap()); + HoodieDeleteBlock deleteBlock = new HoodieDeleteBlock(Option.of(writtenDeleteBlock.getContentBytes(null).toByteArray()), + null, false, Option.empty(), Collections.emptyMap(), Collections.emptyMap()); + assertEquals(Integer.valueOf(deleteEpochDay), deleteBlock.getRecordsToDelete()[0].getOrderingValue()); + + KeyBasedFileGroupRecordBuffer<IndexedRecord> buffer = buildKeyBasedFileGroupRecordBuffer( + readerContext, tableConfig, new HoodieReadStats(), null, RecordMergeMode.EVENT_TIME_ORDERING, + Collections.singletonList("ts"), props, Option.empty()); + try { + buffer.setBaseFileIterator(ClosableIterator.wrap(Collections.<IndexedRecord>emptyList().iterator())); + if (deleteFirst) { + buffer.processDeleteBlock(deleteBlock); + buffer.processDataBlock(dataBlock, Option.empty()); + } else { + buffer.processDataBlock(dataBlock, Option.empty()); + buffer.processDeleteBlock(deleteBlock); + } + // An older delete loses and a newer delete wins in either order; equal event times keep the last arrival. + assertEquals(expectDeleted ? Collections.emptyList() : Collections.singletonList(record), getActualRecords(buffer)); + } finally { + buffer.close(); + } + } + @Test void readWithCommitTimeOrdering() throws IOException { HoodieReadStats readStats = new HoodieReadStats(); HoodieTableConfig tableConfig = mock(HoodieTableConfig.class); + when(tableConfig.getPayloadClassIfPresent()).thenReturn(Option.empty()); when(tableConfig.getRecordKeyFields()).thenReturn(Option.of(new String[] {"record_key"})); when(tableConfig.getPartitionFields()).thenReturn(Option.empty()); StorageConfiguration<?> storageConfiguration = mock(StorageConfiguration.class); @@ -187,6 +268,7 @@ class TestKeyBasedFileGroupRecordBuffer extends BaseTestFileGroupRecordBuffer { void readWithCustomPayload() throws IOException { HoodieReadStats readStats = new HoodieReadStats(); HoodieTableConfig tableConfig = mock(HoodieTableConfig.class); + when(tableConfig.getPayloadClassIfPresent()).thenReturn(Option.empty()); when(tableConfig.getPayloadClass()).thenReturn(CustomPayload.class.getName()); when(tableConfig.getRecordKeyFields()).thenReturn(Option.of(new String[] {"record_key"})); when(tableConfig.getPartitionFields()).thenReturn(Option.empty()); @@ -227,6 +309,7 @@ class TestKeyBasedFileGroupRecordBuffer extends BaseTestFileGroupRecordBuffer { void readWithCustomMerger() throws IOException { HoodieReadStats readStats = new HoodieReadStats(); HoodieTableConfig tableConfig = mock(HoodieTableConfig.class); + when(tableConfig.getPayloadClassIfPresent()).thenReturn(Option.empty()); when(tableConfig.getPayloadClass()).thenReturn(CustomPayload.class.getName()); when(tableConfig.getRecordKeyFields()).thenReturn(Option.of(new String[] {"record_key"})); when(tableConfig.getPartitionFields()).thenReturn(Option.empty()); diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestSortedKeyBasedFileGroupRecordBuffer.java b/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestSortedKeyBasedFileGroupRecordBuffer.java index d1418101c13e..a8bbfa08beb6 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestSortedKeyBasedFileGroupRecordBuffer.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestSortedKeyBasedFileGroupRecordBuffer.java @@ -127,6 +127,7 @@ class TestSortedKeyBasedFileGroupRecordBuffer extends BaseTestFileGroupRecordBuf properties.setProperty(DELETE_KEY, "counter"); properties.setProperty(DELETE_MARKER, "3"); HoodieTableConfig tableConfig = mock(HoodieTableConfig.class); + when(tableConfig.getPayloadClassIfPresent()).thenReturn(Option.empty()); when(tableConfig.getRecordKeyFields()).thenReturn(Option.of(new String[] {"record_key"})); when(tableConfig.getPartitionFields()).thenReturn(Option.empty()); when(tableConfig.getTableVersion()).thenReturn(HoodieTableVersion.current()); diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestStreamingKeyBasedFileGroupRecordBuffer.java b/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestStreamingKeyBasedFileGroupRecordBuffer.java index d03be553194a..5e652d7965ce 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestStreamingKeyBasedFileGroupRecordBuffer.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestStreamingKeyBasedFileGroupRecordBuffer.java @@ -74,6 +74,7 @@ class TestStreamingKeyBasedFileGroupRecordBuffer extends BaseTestFileGroupRecord properties.setProperty(DELETE_KEY, "counter"); properties.setProperty(DELETE_MARKER, "3"); HoodieTableConfig tableConfig = mock(HoodieTableConfig.class); + when(tableConfig.getPayloadClassIfPresent()).thenReturn(Option.empty()); when(tableConfig.getRecordMergeMode()).thenReturn(RecordMergeMode.EVENT_TIME_ORDERING); when(tableConfig.getPartialUpdateMode()).thenReturn(Option.empty()); when(tableConfig.getTableVersion()).thenReturn(HoodieTableVersion.current()); @@ -113,6 +114,7 @@ class TestStreamingKeyBasedFileGroupRecordBuffer extends BaseTestFileGroupRecord properties.setProperty(DELETE_KEY, "counter"); properties.setProperty(DELETE_MARKER, "3"); HoodieTableConfig tableConfig = mock(HoodieTableConfig.class); + when(tableConfig.getPayloadClassIfPresent()).thenReturn(Option.empty()); when(tableConfig.getRecordMergeMode()).thenReturn(RecordMergeMode.COMMIT_TIME_ORDERING); when(tableConfig.getPartialUpdateMode()).thenReturn(Option.empty()); when(tableConfig.getTableVersion()).thenReturn(HoodieTableVersion.current()); @@ -152,6 +154,7 @@ class TestStreamingKeyBasedFileGroupRecordBuffer extends BaseTestFileGroupRecord properties.setProperty(HoodieTableConfig.PAYLOAD_CLASS_NAME.key(), CustomPayload.class.getName()); properties.setProperty(HoodieTableConfig.RECORD_MERGE_STRATEGY_ID.key(), HoodieRecordMerger.PAYLOAD_BASED_MERGE_STRATEGY_UUID); HoodieTableConfig tableConfig = mock(HoodieTableConfig.class); + when(tableConfig.getPayloadClassIfPresent()).thenReturn(Option.empty()); when(tableConfig.getPayloadClass()).thenReturn(CustomPayload.class.getName()); when(tableConfig.getRecordKeyFields()).thenReturn(Option.of(new String[] {"record_key"})); when(tableConfig.getPartitionFields()).thenReturn(Option.empty()); @@ -190,6 +193,7 @@ class TestStreamingKeyBasedFileGroupRecordBuffer extends BaseTestFileGroupRecord properties.setProperty(DELETE_MARKER, "3"); properties.setProperty(HoodieTableConfig.PAYLOAD_CLASS_NAME.key(), CustomPayload.class.getName()); HoodieTableConfig tableConfig = mock(HoodieTableConfig.class); + when(tableConfig.getPayloadClassIfPresent()).thenReturn(Option.empty()); when(tableConfig.getPayloadClass()).thenReturn(CustomPayload.class.getName()); when(tableConfig.getRecordKeyFields()).thenReturn(Option.of(new String[] {"record_key"})); when(tableConfig.getPartitionFields()).thenReturn(Option.empty()); diff --git a/hudi-common/src/test/java/org/apache/hudi/metadata/TestHoodieBackedTableMetadataDataCleanup.java b/hudi-common/src/test/java/org/apache/hudi/metadata/TestHoodieBackedTableMetadataDataCleanup.java index 27a49bf897f7..0854fcea85a9 100644 --- a/hudi-common/src/test/java/org/apache/hudi/metadata/TestHoodieBackedTableMetadataDataCleanup.java +++ b/hudi-common/src/test/java/org/apache/hudi/metadata/TestHoodieBackedTableMetadataDataCleanup.java @@ -602,6 +602,7 @@ public class TestHoodieBackedTableMetadataDataCleanup { when(timeline.filterCompletedInstants()).thenReturn(timeline); when(timeline.lastInstant()).thenReturn(Option.empty()); HoodieTableConfig tableConfig = mock(HoodieTableConfig.class); + when(tableConfig.getPayloadClassIfPresent()).thenReturn(Option.empty()); when(tableConfig.isRecordKeyPopulated()).thenReturn(true); when(tableConfig.getBaseFileFormat()).thenReturn(HoodieFileFormat.PARQUET); when(metadataMetaClient.getTableConfig()).thenReturn(tableConfig); diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBufferedRecordMerger.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBufferedRecordMerger.java index d074cff217ff..65a6ecea2e0b 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBufferedRecordMerger.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBufferedRecordMerger.java @@ -617,6 +617,7 @@ class TestBufferedRecordMerger extends SparkClientFunctionalTestHarness { payloadClassName = Option.empty(); } // Configure reader context with custom schema + when(tableConfig.getPayloadClassIfPresent()).thenReturn(Option.empty()); props.setProperty(ORDERING_FIELDS.key(), "timestamp"); HoodieAvroReaderContext avroReaderContext = new HoodieAvroReaderContext(storageConfig, tableConfig, Option.empty(), Option.empty()); avroReaderContext.setHasLogFiles(false); diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/others/TestMergeModeEventTimeOrdering.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/others/TestMergeModeEventTimeOrdering.scala index 1c61a85fcf81..e898afc7d4ab 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/others/TestMergeModeEventTimeOrdering.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/others/TestMergeModeEventTimeOrdering.scala @@ -23,6 +23,7 @@ import org.apache.hudi.DataSourceWriteOptions import org.apache.hudi.DataSourceWriteOptions.SPARK_SQL_INSERT_INTO_OPERATION import org.apache.hudi.common.config.RecordMergeMode.EVENT_TIME_ORDERING import org.apache.hudi.common.model.DefaultHoodieRecordPayload +import org.apache.hudi.common.model.HoodieRecord.HoodieRecordType import org.apache.hudi.common.model.HoodieRecordMerger.EVENT_TIME_BASED_MERGE_STRATEGY_UUID import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableVersion} import org.apache.hudi.common.testutils.HoodieTestUtils @@ -31,6 +32,36 @@ import org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase import org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase.validateTableConfig class TestMergeModeEventTimeOrdering extends HoodieSparkSqlTestBase { + Seq( + ("date", "date'2026-10-01'", "date'2026-10-02'", false), + ("timestamp", "timestamp'2026-10-01 00:00:00'", "timestamp'2026-10-02 00:00:00'", false), + ("timestamp", "timestamp'2026-10-01 00:00:00'", "timestamp'2026-10-02 00:00:00'", true), + ("decimal(10,2)", "cast(1.50 as decimal(10,2))", "cast(2.50 as decimal(10,2))", false) + ).foreach { case (orderingType, oldValue, newValue, consistentTimestamp) => + test(s"COW SQL upsert with $orderingType ordering field (consistentTimestamp=$consistentTimestamp)") { + withSQLConf(SPARK_SQL_INSERT_INTO_OPERATION.key -> "upsert", + "hoodie.datasource.write.keygenerator.consistent.logical.timestamp.enabled" -> consistentTimestamp.toString) { + withRecordType(Seq(HoodieRecordType.AVRO)) { + withTempDir { tmp => + val tableName = generateTableName + spark.sql( + s""" + | create table $tableName (id int, v int, ts $orderingType) + | using hudi + | tblproperties (type = 'cow', primaryKey = 'id', preCombineField = 'ts') + | location '${tmp.getCanonicalPath}' + """.stripMargin) + spark.sql(s"insert into $tableName values (1, 1, $oldValue)") + spark.sql(s"insert into $tableName values (1, 2, $newValue)") + checkAnswer(s"select id, v from $tableName")(Seq(1, 2)) + spark.sql(s"insert into $tableName values (1, 3, $oldValue)") + checkAnswer(s"select id, v from $tableName")(Seq(1, 2)) + } + } + } + } + } + Seq( "cow,current,true", "cow,current,false", diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/others/TestUpdateTable.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/others/TestUpdateTable.scala index bee62c001858..9a63eced1354 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/others/TestUpdateTable.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/others/TestUpdateTable.scala @@ -34,6 +34,53 @@ import org.junit.jupiter.api.Assertions.assertEquals class TestUpdateTable extends HoodieSparkSqlTestBase { + // (ordering field type, older literal, newer literal, older value as string, newer value as string) + private val logicalTypeOrderingCases = Seq( + ("timestamp", "timestamp'2026-10-01 00:00:00'", "timestamp'2026-10-02 00:00:00'", + "2026-10-01 00:00:00", "2026-10-02 00:00:00"), + ("date", "date'2026-10-01'", "date'2026-10-02'", "2026-10-01", "2026-10-02"), + ("decimal(10,2)", "cast(1.50 as decimal(10,2))", "cast(2.50 as decimal(10,2))", "1.50", "2.50")) + + logicalTypeOrderingCases.foreach { case (orderingType, oldLiteral, newLiteral, oldStr, newStr) => + test(s"Test Update Table With ${orderingType.toUpperCase} Ordering Field") { + withTempDir { tmp => + Seq(true, false).foreach { sparkSqlOptimizedWrites => + val tableName = generateTableName + spark.sql( + s""" + |create table $tableName ( + | id int, + | v int, + | ts $orderingType, + | dt string + |) using hudi + | location '${tmp.getCanonicalPath}/$tableName' + | tblproperties ( + | type = 'cow', + | primaryKey = 'id', + | preCombineField = 'ts' + | ) + """.stripMargin) + spark.sql(s"insert into $tableName values (1, 1, $oldLiteral, 'p1'), (2, 2, $oldLiteral, 'p1')") + + withSQLConf(SPARK_SQL_OPTIMIZED_WRITES.key() -> sparkSqlOptimizedWrites.toString) { + spark.sql(s"update $tableName set v = v * 10 where id = 1") + checkAnswer(s"select id, v, cast(ts as string), dt from $tableName order by id")( + Seq(1, 10, oldStr, "p1"), + Seq(2, 2, oldStr, "p1") + ) + + spark.sql(s"update $tableName set v = v + 1, ts = $newLiteral where id = 2") + checkAnswer(s"select id, v, cast(ts as string), dt from $tableName order by id")( + Seq(1, 10, oldStr, "p1"), + Seq(2, 3, newStr, "p1") + ) + } + } + } + } + } + test("Test Update Table") { withRecordType()(withTempDir { tmp => Seq(true, false).foreach { sparkSqlOptimizedWrites => diff --git a/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiPageSourceProvider.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiPageSourceProvider.java index aafc7ea1c5ff..cee3c74dba07 100644 --- a/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiPageSourceProvider.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiPageSourceProvider.java @@ -279,7 +279,8 @@ public class HudiPageSourceProvider dataPageSource, readColumnHandles, prefilledColumnValues, - logPageSourceFactory); + logPageSourceFactory, + readerProps); Schema requestedSchema = constructSchema(dataSchema.toAvroSchema(), readColumnHandles.stream().map(HiveColumnHandle::getName).toList()); FileSlice fileSlice = convertToFileSlice(hudiSplit, hudiTableHandle.getBasePath()); diff --git a/hudi-trino/src/main/java/io/trino/plugin/hudi/reader/HudiTrinoReaderContext.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/reader/HudiTrinoReaderContext.java index cc812a9a466f..36212bf21d29 100644 --- a/hudi-trino/src/main/java/io/trino/plugin/hudi/reader/HudiTrinoReaderContext.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/reader/HudiTrinoReaderContext.java @@ -25,6 +25,7 @@ import io.trino.spi.connector.SourcePage; import org.apache.avro.generic.IndexedRecord; import org.apache.hudi.common.avro.AvroRecordContext; import org.apache.hudi.common.config.RecordMergeMode; +import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.engine.EngineType; import org.apache.hudi.common.engine.HoodieReaderContext; import org.apache.hudi.common.fs.FSUtils; @@ -34,6 +35,8 @@ import org.apache.hudi.common.model.OverwriteWithLatestMerger; import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.schema.HoodieSchemaField; import org.apache.hudi.common.table.HoodieTableConfig; +import org.apache.hudi.common.table.read.FileGroupReaderSchemaHandler; +import org.apache.hudi.common.util.ConfigUtils; import org.apache.hudi.common.util.HoodieRecordUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.ClosableIterator; @@ -83,9 +86,11 @@ public class HudiTrinoReaderContext ConnectorPageSource pageSource, List<HiveColumnHandle> columnHandles, PrefilledColumnValues prefilledColumnValues, - LogFileParquetPageSourceFactory logPageSourceFactory) + LogFileParquetPageSourceFactory logPageSourceFactory, + TypedProperties props) { - super(storageConfiguration, tableConfig, Option.empty(), Option.empty(), new AvroRecordContext(tableConfig, tableConfig.getPayloadClass())); + super(storageConfiguration, tableConfig, Option.empty(), Option.empty(), + new AvroRecordContext(tableConfig, tableConfig.getPayloadClass(), ConfigUtils.getMergeProps(props, tableConfig))); this.pageSource = pageSource; this.prefilledColumnValues = prefilledColumnValues; this.columnHandles = columnHandles; @@ -100,6 +105,14 @@ public class HudiTrinoReaderContext this.baseProjectionNames = ImmutableSet.copyOf(colNameToHandle.keySet()); } + @Override + public void setSchemaHandler(FileGroupReaderSchemaHandler<IndexedRecord> schemaHandler) + { + super.setSchemaHandler(schemaHandler); + List<String> orderingFieldNames = HoodieRecordUtils.getOrderingFieldNames(getMergeMode(), tableConfig); + ((AvroRecordContext) recordContext).initOrderingValueConverter(schemaHandler.getTableSchema(), orderingFieldNames); + } + @Override public ClosableIterator<IndexedRecord> getFileRecordIterator( StoragePath storagePath, diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/reader/TestHudiTrinoReaderContext.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/reader/TestHudiTrinoReaderContext.java new file mode 100644 index 000000000000..7bf3ca3763a5 --- /dev/null +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/reader/TestHudiTrinoReaderContext.java @@ -0,0 +1,146 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hudi.reader; + +import io.trino.plugin.hudi.storage.TrinoStorageConfiguration; +import io.trino.spi.connector.EmptyPageSource; +import org.apache.avro.LogicalTypes; +import org.apache.avro.Schema; +import org.apache.avro.SchemaBuilder; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.generic.IndexedRecord; +import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.common.model.DeleteRecord; +import org.apache.hudi.common.schema.HoodieSchema; +import org.apache.hudi.common.table.HoodieTableConfig; +import org.apache.hudi.common.table.log.block.HoodieDeleteBlock; +import org.apache.hudi.common.table.read.BufferedRecord; +import org.apache.hudi.common.table.read.BufferedRecordMerger; +import org.apache.hudi.common.table.read.BufferedRecordMergerFactory; +import org.apache.hudi.common.table.read.BufferedRecords; +import org.apache.hudi.common.table.read.FileGroupReaderSchemaHandler; +import org.apache.hudi.common.table.read.TableState; +import org.apache.hudi.common.util.ConfigUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.storage.StoragePath; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +import java.io.IOException; +import java.sql.Timestamp; +import java.util.List; +import java.util.Map; + +import static org.apache.hudi.common.config.RecordMergeMode.EVENT_TIME_ORDERING; +import static org.apache.hudi.common.table.HoodieTableConfig.RECORD_MERGE_PROPERTY_PREFIX; +import static org.apache.hudi.keygen.constant.KeyGeneratorOptions.KEYGENERATOR_CONSISTENT_LOGICAL_TIMESTAMP_ENABLED; +import static org.assertj.core.api.Assertions.assertThat; + +class TestHudiTrinoReaderContext +{ + @ParameterizedTest + @CsvSource({"9, false", "11, true"}) + void testDateOrderingWithSerializedDelete(int deleteEpochDay, boolean expectDeleted) + throws IOException + { + HoodieSchema schema = schemaWithOrdering(LogicalTypes.date().addToSchema(Schema.create(Schema.Type.INT))); + TypedProperties props = new TypedProperties(); + HudiTrinoReaderContext readerContext = createReaderContext(schema, tableConfig(), props); + GenericRecord record = new GenericData.Record(schema.toAvroSchema()); + record.put("id", "key"); + record.put("ordering", 10); + BufferedRecord<IndexedRecord> data = BufferedRecords.fromEngineRecord( + record, "key", schema, readerContext.getRecordContext(), List.of("ordering"), null); + BufferedRecord<IndexedRecord> delete = readDelete(readerContext, deleteEpochDay); + BufferedRecordMerger<IndexedRecord> merger = BufferedRecordMergerFactory.create( + readerContext, EVENT_TIME_ORDERING, false, Option.empty(), schema, Option.empty(), props, Option.empty()); + + assertThat(merger.finalMerge(data, delete)).isSameAs(expectDeleted ? delete : data); + } + + @ParameterizedTest + @CsvSource({"false, false", "true, false", "false, true", "true, true"}) + void testTimestampOrderingConfiguration(boolean consistentTimestamp, boolean persistedProperty) + throws IOException + { + HoodieSchema schema = schemaWithOrdering(LogicalTypes.timestampMicros().addToSchema(Schema.create(Schema.Type.LONG))); + HoodieTableConfig tableConfig = tableConfig(); + TypedProperties props = new TypedProperties(); + String key = KEYGENERATOR_CONSISTENT_LOGICAL_TIMESTAMP_ENABLED.key(); + if (persistedProperty) { + tableConfig.setValue(RECORD_MERGE_PROPERTY_PREFIX + key, Boolean.toString(consistentTimestamp)); + // Persisted merge properties take precedence over the reader's value. + props.setProperty(key, Boolean.toString(!consistentTimestamp)); + } + else { + props.setProperty(key, Boolean.toString(consistentTimestamp)); + } + HudiTrinoReaderContext readerContext = createReaderContext(schema, tableConfig, props); + GenericRecord record = new GenericData.Record(schema.toAvroSchema()); + record.put("id", "key"); + record.put("ordering", 1_234_567L); + Object expected = consistentTimestamp ? new Timestamp(1234) : 1_234_567L; + + assertThat(readerContext.getRecordContext().getOrderingValue(record, schema, List.of("ordering"))).isEqualTo(expected); + assertThat(readDelete(readerContext, 1_234_567L).getOrderingValue()).isEqualTo(expected); + } + + private static HoodieSchema schemaWithOrdering(Schema orderingSchema) + { + return HoodieSchema.fromAvroSchema(SchemaBuilder.record("test_record").fields() + .requiredString("id") + .name("ordering").type(orderingSchema).noDefault() + .endRecord()); + } + + private static HoodieTableConfig tableConfig() + { + HoodieTableConfig tableConfig = new HoodieTableConfig(); + tableConfig.setValue(HoodieTableConfig.VERSION, "9"); + tableConfig.setValue(HoodieTableConfig.RECORD_MERGE_MODE, EVENT_TIME_ORDERING.name()); + tableConfig.setValue(HoodieTableConfig.ORDERING_FIELDS, "ordering"); + return tableConfig; + } + + private static HudiTrinoReaderContext createReaderContext(HoodieSchema schema, HoodieTableConfig tableConfig, TypedProperties props) + { + // These tests consume classic log records directly; no page source is read. + HudiTrinoReaderContext readerContext = new HudiTrinoReaderContext( + new TrinoStorageConfiguration(), tableConfig, new EmptyPageSource(), List.of(), null, + (path, start, length, projection) -> { + throw new AssertionError("Unexpected parquet read"); + }, props); + TypedProperties mergeProps = ConfigUtils.getMergeProps(props, tableConfig); + readerContext.initRecordMerger(mergeProps); + readerContext.setHasLogFiles(false); + readerContext.setHasBootstrapBaseFile(false); + HoodieSchema projectedSchema = HoodieSchema.fromAvroSchema(SchemaBuilder.record("test_record").fields().requiredString("id").endRecord()); + // The ordering field can be absent from the projection, so initialization must use the table schema. + readerContext.setSchemaHandler(new FileGroupReaderSchemaHandler<>( + readerContext, schema, projectedSchema, Option.empty(), mergeProps, + TableState.of(new StoragePath("/test-table"), tableConfig, Option.empty(), Option.empty()))); + return readerContext; + } + + private static BufferedRecord<IndexedRecord> readDelete(HudiTrinoReaderContext readerContext, Comparable<?> orderingValue) + throws IOException + { + HoodieDeleteBlock written = new HoodieDeleteBlock(List.of(Pair.of(DeleteRecord.create("key", "", orderingValue), -1L)), Map.of()); + HoodieDeleteBlock read = new HoodieDeleteBlock(Option.of(written.getContentBytes(null).toByteArray()), + null, false, Option.empty(), Map.of(), Map.of()); + return read.getRecordsToDelete(readerContext).getFirst(); + } +}
