This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 3affaf7dd232 fix(hive-sync): pass the default partition through the
slash-encoded value extractors (#19710)
3affaf7dd232 is described below
commit 3affaf7dd232fc1d1a5eeb035ca33a04b032b969
Author: Sepuri Sai Krishna <[email protected]>
AuthorDate: Tue Aug 25 15:45:00 2026 +0530
fix(hive-sync): pass the default partition through the slash-encoded value
extractors (#19710)
SlashEncodedDayPartitionValueExtractor requires a partition path of exactly
three segments, but a null partition value is written to the single-segment
__HIVE_DEFAULT_PARTITION__ directory. Since that extractor is auto-inferred
for
tables using slash.separated.date.partitioning, hive sync of any such table
that has seen a null partition value fails outright.
Return the default partition marker as-is, which is what Hive uses for a
null
partition value, before the segment-count check. Apply the same to
SlashEncodedHourPartitionValueExtractor, which carries the identical defect
against its four-segment check.
Closes #19668
---
.../hudi/common/util/PartitionPathEncodeUtils.java | 13 +++++++
.../SlashEncodedDayPartitionValueExtractor.java | 10 +++++
.../SlashEncodedHourPartitionValueExtractor.java | 10 +++++
.../hudi/hive/TestPartitionValueExtractor.java | 44 ++++++++++++++++++++++
4 files changed, 77 insertions(+)
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/util/PartitionPathEncodeUtils.java
b/hudi-common/src/main/java/org/apache/hudi/common/util/PartitionPathEncodeUtils.java
index 83349cffee5f..b3499fd75f26 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/util/PartitionPathEncodeUtils.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/util/PartitionPathEncodeUtils.java
@@ -29,6 +29,19 @@ public class PartitionPathEncodeUtils {
public static final String DEPRECATED_DEFAULT_PARTITION_PATH = "default";
public static final String DEFAULT_PARTITION_PATH =
"__HIVE_DEFAULT_PARTITION__";
+ /**
+ * Returns whether {@code partitionValue} is the marker Hudi writes for a
null or empty partition
+ * value. Both the current marker and the pre-0.12 {@code default} one are
recognised, matching
+ * {@code PartitionPathParser#parseValue}. The value may be a whole
partition directory, in which
+ * case a Hive-style column prefix (e.g. {@code
datestr=__HIVE_DEFAULT_PARTITION__}) is stripped
+ * before comparing.
+ */
+ public static boolean isDefaultPartitionValue(String partitionValue) {
+ int separator = partitionValue.indexOf('=');
+ String value = separator < 0 ? partitionValue :
partitionValue.substring(separator + 1);
+ return DEFAULT_PARTITION_PATH.equals(value) ||
DEPRECATED_DEFAULT_PARTITION_PATH.equals(value);
+ }
+
static BitSet charToEscape = new BitSet(128);
static BitSet charToEscapeFilename = new BitSet(128);
static {
diff --git
a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/SlashEncodedDayPartitionValueExtractor.java
b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/SlashEncodedDayPartitionValueExtractor.java
index 85fda472754e..6ad920aa80ae 100644
---
a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/SlashEncodedDayPartitionValueExtractor.java
+++
b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/SlashEncodedDayPartitionValueExtractor.java
@@ -28,6 +28,9 @@ import java.time.format.DateTimeFormatter;
import java.util.Collections;
import java.util.List;
+import static
org.apache.hudi.common.util.PartitionPathEncodeUtils.DEFAULT_PARTITION_PATH;
+import static
org.apache.hudi.common.util.PartitionPathEncodeUtils.isDefaultPartitionValue;
+
/**
* HDFS Path contain hive partition values for the keys it is partitioned on.
This mapping is not straight forward and
* requires a pluggable implementation to extract the partition value from
HDFS path.
@@ -52,6 +55,13 @@ public class SlashEncodedDayPartitionValueExtractor
implements PartitionValueExt
@Override
public List<String> extractPartitionValuesInPath(String partitionPath) {
+ // NOTE: A null or empty partition value lands in the default-partition
directory, which is a
+ // single segment rather than the yyyy/mm/dd layout it would
otherwise be. Hive uses this
+ // same marker for a null partition value, so it is passed straight
through. The
+ // pre-0.12 "default" marker is recognised too, matching
PartitionPathParser#parseValue
+ if (isDefaultPartitionValue(partitionPath)) {
+ return Collections.singletonList(DEFAULT_PARTITION_PATH);
+ }
// partition path is expected to be in this format yyyy/mm/dd
String[] splits = partitionPath.split("/");
if (splits.length != 3) {
diff --git
a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/SlashEncodedHourPartitionValueExtractor.java
b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/SlashEncodedHourPartitionValueExtractor.java
index a30296b403d7..767e3538b045 100644
---
a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/SlashEncodedHourPartitionValueExtractor.java
+++
b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/SlashEncodedHourPartitionValueExtractor.java
@@ -28,6 +28,9 @@ import java.time.format.DateTimeFormatter;
import java.util.Collections;
import java.util.List;
+import static
org.apache.hudi.common.util.PartitionPathEncodeUtils.DEFAULT_PARTITION_PATH;
+import static
org.apache.hudi.common.util.PartitionPathEncodeUtils.isDefaultPartitionValue;
+
/**
* HDFS Path contain hive partition values for the keys it is partitioned on.
This mapping is not straight forward and
* requires a pluggable implementation to extract the partition value from
HDFS path.
@@ -52,6 +55,13 @@ public class SlashEncodedHourPartitionValueExtractor
implements PartitionValueEx
@Override
public List<String> extractPartitionValuesInPath(String partitionPath) {
+ // NOTE: A null or empty partition value lands in the default-partition
directory, which is a
+ // single segment rather than the yyyy/mm/dd/HH layout it would
otherwise be. Hive uses this
+ // same marker for a null partition value, so it is passed straight
through. The
+ // pre-0.12 "default" marker is recognised too, matching
PartitionPathParser#parseValue
+ if (isDefaultPartitionValue(partitionPath)) {
+ return Collections.singletonList(DEFAULT_PARTITION_PATH);
+ }
// partition path is expected to be in this format yyyy/mm/dd/HH
String[] splits = partitionPath.split("/");
if (splits.length != 4) {
diff --git
a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestPartitionValueExtractor.java
b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestPartitionValueExtractor.java
index 0cce6ee79423..fb9cb8803392 100644
---
a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestPartitionValueExtractor.java
+++
b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestPartitionValueExtractor.java
@@ -26,6 +26,8 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import static
org.apache.hudi.common.util.PartitionPathEncodeUtils.DEFAULT_PARTITION_PATH;
+import static
org.apache.hudi.common.util.PartitionPathEncodeUtils.DEPRECATED_DEFAULT_PARTITION_PATH;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -38,6 +40,48 @@ public class TestPartitionValueExtractor {
assertEquals(hourPartition.extractPartitionValuesInPath("2020/12/20/01"),
list);
assertThrows(IllegalArgumentException.class, () ->
hourPartition.extractPartitionValuesInPath("2020/12/20"));
assertEquals(hourPartition.extractPartitionValuesInPath("update_time=2020/12/20/01"),
list);
+ // a null partition value is written to the single-segment
default-partition directory rather
+ // than the yyyy/mm/dd/HH layout, and has to survive the extractor rather
than blow it up
+ assertEquals(
+ Collections.singletonList(DEFAULT_PARTITION_PATH),
+ hourPartition.extractPartitionValuesInPath(DEFAULT_PARTITION_PATH));
+ assertEquals(
+ Collections.singletonList(DEFAULT_PARTITION_PATH),
+ hourPartition.extractPartitionValuesInPath("update_time=" +
DEFAULT_PARTITION_PATH));
+ // the pre-0.12 marker lands in a "default" directory and maps to the same
Hive null marker
+ assertEquals(
+ Collections.singletonList(DEFAULT_PARTITION_PATH),
+
hourPartition.extractPartitionValuesInPath(DEPRECATED_DEFAULT_PARTITION_PATH));
+ assertEquals(
+ Collections.singletonList(DEFAULT_PARTITION_PATH),
+ hourPartition.extractPartitionValuesInPath("update_time=" +
DEPRECATED_DEFAULT_PARTITION_PATH));
+ }
+
+ @Test
+ public void testDayPartition() {
+ SlashEncodedDayPartitionValueExtractor dayPartition = new
SlashEncodedDayPartitionValueExtractor();
+ assertEquals(
+ Collections.singletonList("2026-01-05"),
+ dayPartition.extractPartitionValuesInPath("2026/01/05"));
+ assertEquals(
+ Collections.singletonList("2026-01-05"),
+ dayPartition.extractPartitionValuesInPath("datestr=2026/01/05"));
+ assertThrows(IllegalArgumentException.class, () ->
dayPartition.extractPartitionValuesInPath("2026/01"));
+ // a null partition value is written to the single-segment
default-partition directory rather
+ // than the yyyy/mm/dd layout, and has to survive the extractor rather
than blow it up
+ assertEquals(
+ Collections.singletonList(DEFAULT_PARTITION_PATH),
+ dayPartition.extractPartitionValuesInPath(DEFAULT_PARTITION_PATH));
+ assertEquals(
+ Collections.singletonList(DEFAULT_PARTITION_PATH),
+ dayPartition.extractPartitionValuesInPath("datestr=" +
DEFAULT_PARTITION_PATH));
+ // the pre-0.12 marker lands in a "default" directory and maps to the same
Hive null marker
+ assertEquals(
+ Collections.singletonList(DEFAULT_PARTITION_PATH),
+
dayPartition.extractPartitionValuesInPath(DEPRECATED_DEFAULT_PARTITION_PATH));
+ assertEquals(
+ Collections.singletonList(DEFAULT_PARTITION_PATH),
+ dayPartition.extractPartitionValuesInPath("datestr=" +
DEPRECATED_DEFAULT_PARTITION_PATH));
}
@Test
