This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new cc75051fc4ef fix(hive-sync): fix partition-value parsing on '=' and
pushdown comparator overflow (#19336)
cc75051fc4ef is described below
commit cc75051fc4efa5449f1e0d9e6f6bffe29e2b14f6
Author: Vova Kolmakov <[email protected]>
AuthorDate: Tue Jul 21 23:45:52 2026 +0700
fix(hive-sync): fix partition-value parsing on '=' and pushdown comparator
overflow (#19336)
Two edge-case defects in the partition path of hudi-hive-sync:
- PartitionFilterGenerator.ValueComparator sorted int/bigint partition
values with subtraction (i1 - i2, Long.signum(l1 - l2)) to derive the pushdown
min/max bounds. The subtraction overflows when the difference exceeds the type
range (e.g. Integer.MAX_VALUE and a negative value), giving a wrong ordering
and wrong bounds that can exclude valid partitions or trip the Comparator
contract check. Use Integer.compare/Long.compare.
- MultiPartKeysValueExtractor and HiveStylePartitionValueExtractor split a
hive-style key=value segment with split("=") (no limit), so a value that itself
contains '=' (base64 padding or an embedded '=') was rejected (aborting the
sync) or silently truncated (trailing '=' dropped), causing drift from the
metastore. Split on the first '=' only with split("=", 2).
Extends TestPartitionFilterGenerator (extreme int/bigint min/max bounds),
TestMultiPartKeysValueExtractor and TestPartitionValueExtractor (values
containing '=').
Co-authored-by: Vova Kolmakov <[email protected]>
---
.../hive/HiveStylePartitionValueExtractor.java | 4 +++-
.../hudi/hive/MultiPartKeysValueExtractor.java | 4 +++-
.../hudi/hive/util/PartitionFilterGenerator.java | 11 +++++-----
.../hudi/hive/TestMultiPartKeysValueExtractor.java | 15 ++++++++++++++
.../hudi/hive/TestPartitionValueExtractor.java | 8 ++++++++
.../hive/util/TestPartitionFilterGenerator.java | 24 ++++++++++++++++++++++
6 files changed, 58 insertions(+), 8 deletions(-)
diff --git
a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveStylePartitionValueExtractor.java
b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveStylePartitionValueExtractor.java
index 11098698e8ae..41ccaa59d486 100644
---
a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveStylePartitionValueExtractor.java
+++
b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveStylePartitionValueExtractor.java
@@ -34,7 +34,9 @@ public class HiveStylePartitionValueExtractor implements
PartitionValueExtractor
@Override
public List<String> extractPartitionValuesInPath(String partitionPath) {
// partition path is expected to be in this format
partition_key=partition_value.
- String[] splits = partitionPath.split("=");
+ // Split on the first '=' only so a value that itself contains '=' (e.g.
base64 padding)
+ // is kept intact rather than making the split produce more than two parts.
+ String[] splits = partitionPath.split("=", 2);
if (splits.length != 2) {
throw new IllegalArgumentException(
"Partition path " + partitionPath + " is not in the form
partition_key=partition_value.");
diff --git
a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/MultiPartKeysValueExtractor.java
b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/MultiPartKeysValueExtractor.java
index dd356638a47e..5404649a5fd6 100644
---
a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/MultiPartKeysValueExtractor.java
+++
b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/MultiPartKeysValueExtractor.java
@@ -42,7 +42,9 @@ public class MultiPartKeysValueExtractor implements
PartitionValueExtractor {
String[] splits = partitionPath.split("/");
return Arrays.stream(splits).map(s -> {
if (s.contains("=")) {
- String[] moreSplit = s.split("=");
+ // Split on the first '=' only so partition values that themselves
contain '='
+ // (e.g. base64 padding like "col=YWJj==") are preserved instead of
being rejected or truncated.
+ String[] moreSplit = s.split("=", 2);
ValidationUtils.checkArgument(moreSplit.length == 2, "Partition Field
(" + s + ") not in expected format");
return moreSplit[1];
}
diff --git
a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/PartitionFilterGenerator.java
b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/PartitionFilterGenerator.java
index ea64c111d42a..75602bc57708 100644
---
a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/PartitionFilterGenerator.java
+++
b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/PartitionFilterGenerator.java
@@ -140,13 +140,12 @@ public class PartitionFilterGenerator {
public int compare(String s1, String s2) {
switch (valueType.toLowerCase(Locale.ROOT)) {
case HiveSchemaUtil.INT_TYPE_NAME:
- int i1 = Integer.parseInt(s1);
- int i2 = Integer.parseInt(s2);
- return i1 - i2;
+ // Use Integer.compare rather than subtraction, which overflows for
values whose
+ // difference exceeds the int range and yields a wrong ordering.
+ return Integer.compare(Integer.parseInt(s1), Integer.parseInt(s2));
case HiveSchemaUtil.BIGINT_TYPE_NAME:
- long l1 = Long.parseLong(s1);
- long l2 = Long.parseLong(s2);
- return Long.signum(l1 - l2);
+ // Use Long.compare rather than subtraction, which overflows the
long arithmetic.
+ return Long.compare(Long.parseLong(s1), Long.parseLong(s2));
case HiveSchemaUtil.DATE_TYPE_NAME:
case HiveSchemaUtil.STRING_TYPE_NAME:
return s1.compareTo(s2);
diff --git
a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestMultiPartKeysValueExtractor.java
b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestMultiPartKeysValueExtractor.java
index d8b9100309e5..74610a0ea8e6 100644
---
a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestMultiPartKeysValueExtractor.java
+++
b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestMultiPartKeysValueExtractor.java
@@ -21,6 +21,8 @@ package org.apache.hudi.hive;
import org.junit.jupiter.api.Test;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -40,4 +42,17 @@ public class TestMultiPartKeysValueExtractor {
// Test extract hive style partition path
assertEquals(expected,
valueExtractor.extractPartitionValuesInPath("ds=2021-04-25/hh=04"));
}
+
+ @Test
+ public void testValuesContainingEquals() {
+ MultiPartKeysValueExtractor valueExtractor = new
MultiPartKeysValueExtractor();
+ // Only the first '=' separates key from value, so a value containing '='
is kept intact.
+ assertEquals(Collections.singletonList("a=b"),
valueExtractor.extractPartitionValuesInPath("k=a=b"));
+ // base64-encoded value with '=' padding must not be truncated
+ assertEquals(Collections.singletonList("YWJjZA=="),
valueExtractor.extractPartitionValuesInPath("col=YWJjZA=="));
+ // empty value
+ assertEquals(Collections.singletonList(""),
valueExtractor.extractPartitionValuesInPath("dt="));
+ // multiple hive-style parts whose values contain '='
+ assertEquals(Arrays.asList("a=b", "YWJj=="),
valueExtractor.extractPartitionValuesInPath("k1=a=b/k2=YWJj=="));
+ }
}
diff --git
a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestPartitionValueExtractor.java
b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestPartitionValueExtractor.java
index 075542d59671..0cce6ee79423 100644
---
a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestPartitionValueExtractor.java
+++
b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestPartitionValueExtractor.java
@@ -49,6 +49,14 @@ public class TestPartitionValueExtractor {
assertThrows(
IllegalArgumentException.class,
() -> hiveStylePartition.extractPartitionValuesInPath("2021/04/02"));
+ // Only the first '=' is the separator, so a value containing '=' is
preserved.
+ assertEquals(
+ Collections.singletonList("a=b=c"),
+ hiveStylePartition.extractPartitionValuesInPath("k=a=b=c"));
+ // base64-encoded value with '=' padding must not be truncated
+ assertEquals(
+ Collections.singletonList("YWJjZA=="),
+ hiveStylePartition.extractPartitionValuesInPath("col=YWJjZA=="));
}
@Test
diff --git
a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/util/TestPartitionFilterGenerator.java
b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/util/TestPartitionFilterGenerator.java
index b607e7f6948c..a010261a21bd 100644
---
a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/util/TestPartitionFilterGenerator.java
+++
b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/util/TestPartitionFilterGenerator.java
@@ -79,6 +79,30 @@ public class TestPartitionFilterGenerator {
partitionFilterGenerator.generatePushDownFilter(writtenPartitions,
partitionFieldSchemas, config));
}
+ @Test
+ public void testMinMaxFilterNoNumericOverflow() {
+ Properties props = new Properties();
+ // force the min/max branch so the values are sorted by ValueComparator
+ props.put(HIVE_SYNC_FILTER_PUSHDOWN_MAX_SIZE.key(), "0");
+ HiveSyncConfig config = new HiveSyncConfig(props);
+ List<FieldSchema> partitionFieldSchemas = new ArrayList<>(2);
+ partitionFieldSchemas.add(new FieldSchema("intcol", "int"));
+ partitionFieldSchemas.add(new FieldSchema("bigcol", "bigint"));
+
+ List<String> writtenPartitions = new ArrayList<>();
+ // extreme values whose pairwise difference overflows int/long subtraction
+ writtenPartitions.add("2147483647/9223372036854775807");
+ writtenPartitions.add("-2147483648/-9223372036854775808");
+ writtenPartitions.add("0/0");
+
+ // bounds must reflect the true numeric order; a subtraction-based
comparator overflows and
+ // would pick wrong min/max (or throw a comparator-contract violation).
+ assertEquals(
+ "((intcol >= -2147483648 AND intcol <= 2147483647) "
+ + "AND (bigcol >= -9223372036854775808 AND bigcol <=
9223372036854775807))",
+ partitionFilterGenerator.generatePushDownFilter(writtenPartitions,
partitionFieldSchemas, config));
+ }
+
@Test
public void testPushDownFilterIfExceedLimit() {
Properties props = new Properties();
