This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new a7deb61f7426 fix(schema): lowercase projected names with Locale.ROOT
(#19835)
a7deb61f7426 is described below
commit a7deb61f74263d5fa4d1b71fb32b17d70263ada9
Author: voonhous <[email protected]>
AuthorDate: Sat Sep 5 17:09:27 2026 +0800
fix(schema): lowercase projected names with Locale.ROOT (#19835)
* fix(schema): lowercase projected names with Locale.ROOT
HoodieSchemaUtils.generateProjectionSchema lowercased both the schema
field names and the requested names with the JVM default locale, while
HiveHoodieReaderContext pre-lowercases the requested names with
Locale.ROOT. Under a tr_TR or az_AZ default locale an upper-case I in a
column name becomes dotless-i on the schema side only, the lookup
misses and the Hive read fails with "Field id not found in log schema".
The legacy realtime reader path lowercased the same way and is aligned
too: the projection lookup, getNameToFieldMap and addPartitionFields in
HoodieRealtimeRecordReaderUtils, and constructHiveOrderedSchema in
AbstractRealtimeRecordReader. The projection test now uses a name with
an I and a second test pins the lookup under a Turkish default locale.
Closes #19826
* fix(test): point the locale-toggle note at the restore
The comment credited surefire's single-forked, sequential run for
keeping the tr-TR default-locale toggle from leaking. That is the
condition that would let an unrestored toggle reach later tests in
the same JVM, not what prevents it, and forkCount/reuseForks are
pinned only in the functional-tests profiles anyway. Point the note
at the finally restore, which is what actually makes this safe.
---
.../hudi/common/schema/HoodieSchemaUtils.java | 14 ++++++----
.../hudi/common/schema/TestHoodieSchemaUtils.java | 31 ++++++++++++++++++++--
.../realtime/AbstractRealtimeRecordReader.java | 2 +-
.../utils/HoodieRealtimeRecordReaderUtils.java | 9 ++++---
4 files changed, 44 insertions(+), 12 deletions(-)
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaUtils.java
b/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaUtils.java
index ac2de5fe79fc..1c5e74fcd7ed 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaUtils.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaUtils.java
@@ -39,6 +39,7 @@ import org.apache.avro.generic.GenericData;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
@@ -486,9 +487,11 @@ public final class HoodieSchemaUtils {
*
* <p>Field names are matched case-insensitively and the projected field
keeps the schema's original casing:
* Avro field names are case-sensitive while Hive lowercases column
projections before they reach the reader
- * (see {@code HoodieRealtimeRecordReaderUtils#generateProjectionSchema}),
so both sides are lowercased for the
- * lookup. A schema with two fields that differ only in case cannot be
projected and fails on the duplicate
- * lowercase key.</p>
+ * (see {@code HoodieRealtimeRecordReaderUtils#generateProjectionSchema}),
so both sides are lowercased with
+ * {@code Locale.ROOT} for the lookup. The default locale would map an
upper-case I to dotless-i under a Turkish
+ * or Azeri locale and break the match with {@code HiveHoodieReaderContext},
which pre-lowercases with
+ * {@code Locale.ROOT}. A schema with two fields that differ only in case
cannot be projected and fails on the
+ * duplicate lowercase key.</p>
*
* @param originalSchema the source schema
* @param fieldNames the list of field names to include in the projection
@@ -501,10 +504,11 @@ public final class HoodieSchemaUtils {
ValidationUtils.checkArgument(fieldNames != null, "Field names cannot be
null");
Map<String, HoodieSchemaField> schemaFieldsMap =
originalSchema.getFields().stream()
- .map(r -> Pair.of(r.name().toLowerCase(),
r)).collect(Collectors.toMap(Pair::getLeft, Pair::getRight));
+ .map(r -> Pair.of(r.name().toLowerCase(Locale.ROOT), r))
+ .collect(Collectors.toMap(Pair::getLeft, Pair::getRight));
List<HoodieSchemaField> projectedFields = new
ArrayList<>(fieldNames.size());
for (String fn : fieldNames) {
- HoodieSchemaField field = schemaFieldsMap.get(fn.toLowerCase());
+ HoodieSchemaField field =
schemaFieldsMap.get(fn.toLowerCase(Locale.ROOT));
if (field == null) {
throw new HoodieException("Field " + fn + " not found in log schema.
Query cannot proceed! "
+ "Derived Schema Fields: " + new
ArrayList<>(schemaFieldsMap.keySet()));
diff --git
a/hudi-common/src/test/java/org/apache/hudi/common/schema/TestHoodieSchemaUtils.java
b/hudi-common/src/test/java/org/apache/hudi/common/schema/TestHoodieSchemaUtils.java
index 99d1df06055f..e87913c85b8f 100644
---
a/hudi-common/src/test/java/org/apache/hudi/common/schema/TestHoodieSchemaUtils.java
+++
b/hudi-common/src/test/java/org/apache/hudi/common/schema/TestHoodieSchemaUtils.java
@@ -39,6 +39,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Random;
import java.util.Set;
@@ -1046,15 +1047,41 @@ public class TestHoodieSchemaUtils {
assertTrue(fieldNames1.contains("timestamp"));
// Field names are matched case-insensitively; HiveHoodieReaderContext
lowercases names before calling this.
- HoodieSchema schema2 =
HoodieSchemaUtils.generateProjectionSchema(originalSchema,
Arrays.asList("_ROW_KEY"));
+ HoodieSchema schema2 =
HoodieSchemaUtils.generateProjectionSchema(originalSchema,
Arrays.asList("PII_COL"));
assertEquals(1, schema2.getFields().size());
- assertEquals("_row_key", schema2.getFields().get(0).name());
+ assertEquals("pii_col", schema2.getFields().get(0).name());
Throwable caughtException = assertThrows(HoodieException.class, () ->
HoodieSchemaUtils.generateProjectionSchema(originalSchema,
Arrays.asList("_row_key", "timestamp", "fake_field")));
assertTrue(caughtException.getMessage().contains("Field fake_field not
found in log schema. Query cannot proceed!"));
}
+ @Test
+ public void testGenerateProjectionSchemaIgnoresDefaultLocale() {
+ // Under tr-TR, String#toLowerCase() maps an upper-case I to dotless-i
(U+0131), so a default-locale lowercase
+ // on one side of the lookup and Locale.ROOT on the other
(HiveHoodieReaderContext) cannot match for any name
+ // that contains an upper-case I. Surefire reuses one JVM across the
module's tests, so the finally block below
+ // is what keeps the toggle from reaching any test that runs after this
one.
+ Locale saved = Locale.getDefault();
+ Locale.setDefault(Locale.forLanguageTag("tr-TR"));
+ try {
+ HoodieSchema originalSchema = HoodieSchema.parse(EXAMPLE_SCHEMA);
+ // Request upper-cased, schema lower-cased.
+ HoodieSchema projected =
HoodieSchemaUtils.generateProjectionSchema(originalSchema,
Arrays.asList("PII_COL"));
+ assertEquals(1, projected.getFields().size());
+ assertEquals("pii_col", projected.getFields().get(0).name());
+
+ // Schema upper-cased, request pre-lowercased with Locale.ROOT the way
HiveHoodieReaderContext does it.
+ HoodieSchema upperCaseSchema = HoodieSchema.parse("{\"type\":
\"record\",\"name\": \"rec\",\"fields\": ["
+ + "{\"name\": \"ID\", \"type\": \"string\"},{\"name\": \"value\",
\"type\": \"int\"}]}");
+ HoodieSchema projectedUpper =
HoodieSchemaUtils.generateProjectionSchema(upperCaseSchema,
Arrays.asList("id"));
+ assertEquals(1, projectedUpper.getFields().size());
+ assertEquals("ID", projectedUpper.getFields().get(0).name());
+ } finally {
+ Locale.setDefault(saved);
+ }
+ }
+
@Test
public void testAppendFieldsToSchemaDedupNested() {
HoodieSchema fullSchema = HoodieSchema.parse("{\n"
diff --git
a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/AbstractRealtimeRecordReader.java
b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/AbstractRealtimeRecordReader.java
index ce58db552e23..1f01f6ccdce2 100644
---
a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/AbstractRealtimeRecordReader.java
+++
b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/AbstractRealtimeRecordReader.java
@@ -194,7 +194,7 @@ public abstract class AbstractRealtimeRecordReader {
List<HoodieSchemaField> hiveSchemaFields = new ArrayList<>();
for (String columnName : hiveColumns) {
- HoodieSchemaField field = schemaFieldsMap.get(columnName.toLowerCase());
+ HoodieSchemaField field =
schemaFieldsMap.get(columnName.toLowerCase(Locale.ROOT));
if (field != null) {
hiveSchemaFields.add(HoodieSchemaUtils.createNewSchemaField(field));
diff --git
a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieRealtimeRecordReaderUtils.java
b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieRealtimeRecordReaderUtils.java
index c3f643abcda6..70d62cbc0f98 100644
---
a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieRealtimeRecordReaderUtils.java
+++
b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieRealtimeRecordReaderUtils.java
@@ -63,6 +63,7 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.LinkedHashSet;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Random;
import java.util.Set;
@@ -135,7 +136,7 @@ public class HoodieRealtimeRecordReaderUtils {
*/
List<HoodieSchemaField> projectedFields = new ArrayList<>();
for (String fn : fieldNames) {
- HoodieSchemaField field = schemaFieldsMap.get(fn.toLowerCase());
+ HoodieSchemaField field =
schemaFieldsMap.get(fn.toLowerCase(Locale.ROOT));
if (field == null) {
throw new HoodieException("Field " + fn + " not found in log schema.
Query cannot proceed! "
+ "Derived Schema Fields: " + new
ArrayList<>(schemaFieldsMap.keySet()));
@@ -150,7 +151,7 @@ public class HoodieRealtimeRecordReaderUtils {
}
public static Map<String, HoodieSchemaField> getNameToFieldMap(HoodieSchema
schema) {
- return schema.getFields().stream().map(r ->
Pair.of(r.name().toLowerCase(), r))
+ return schema.getFields().stream().map(r ->
Pair.of(r.name().toLowerCase(Locale.ROOT), r))
.collect(Collectors.toMap(Pair::getLeft, Pair::getRight));
}
@@ -311,8 +312,8 @@ public class HoodieRealtimeRecordReaderUtils {
*/
public static HoodieSchema addPartitionFields(HoodieSchema schema,
List<String> partitioningFields) {
final Set<String> firstLevelFieldNames =
-
schema.getFields().stream().map(HoodieSchemaField::name).map(String::toLowerCase).collect(Collectors.toSet());
- List<String> fieldsToAdd =
partitioningFields.stream().map(String::toLowerCase)
+ schema.getFields().stream().map(f ->
f.name().toLowerCase(Locale.ROOT)).collect(Collectors.toSet());
+ List<String> fieldsToAdd = partitioningFields.stream().map(f ->
f.toLowerCase(Locale.ROOT))
.filter(x ->
!firstLevelFieldNames.contains(x)).collect(Collectors.toList());
return appendNullSchemaFields(schema, fieldsToAdd);
