This is an automated email from the ASF dual-hosted git repository.

jerryshao pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git


The following commit(s) were added to refs/heads/main by this push:
     new 68b3efb722 [#14291] fix(spark-connector): Use the Hive default 
partition name for null values and resolve partial specs by name (#14295)
68b3efb722 is described below

commit 68b3efb7227d46e17acc810357033b11599788e9
Author: YangJie <[email protected]>
AuthorDate: Fri Nov 6 06:36:02 2026 -0500

    [#14291] fix(spark-connector): Use the Hive default partition name for null 
values and resolve partial specs by name (#14295)
    
    ### What changes were proposed in this pull request?
    
    - `HiveGravitinoOperationOperator` writes a null partition value as
    `__HIVE_DEFAULT_PARTITION__` in the partition names it builds for drop,
    exists, load and list, and in the values it sends for `ADD PARTITION`.
    The read side already maps this name back to null.
    - The `names` variant of `getHivePartitionName` looks each column up by
    name instead of by position, since Spark passes only the columns of a
    partial spec.
    - `listPartitionIdentifiers` matches whole `column=value` parts of each
    partition name instead of substrings.
    
    ### Why are the changes needed?
    
    Partitions with a null value could not be dropped or found, and `ADD
    PARTITION` with a null value stored the string `null`. A partial spec on
    any column but the first read the value with the wrong column's type,
    which failed with a `ClassCastException` or matched nothing. The
    substring match also let `region=us` match a partition with
    `subregion=us`.
    
    Fix: #14291
    
    ### Does this PR introduce _any_ user-facing change?
    
    `DROP PARTITION (x = null)` and `ADD PARTITION (x = null)` now act on
    the Hive default partition. Partitions added by an earlier connector
    with `ADD PARTITION (x = null)` hold the string `null`; on a string
    column they are now dropped with `x = 'null'`.
    
    ### How was this patch tested?
    
    - `TestHiveGravitinoOperationOperator` adds cases for a null value on
    drop, add, load and exists, in both an `INT` and a `STRING` column, for
    a partial spec on the second column of an `INT, STRING` partitioned
    table, and for a partial spec on `region` in a table that also has a
    `subregion` column.
    - `SparkHiveCatalogIT.testManagePartitionTable` adds `SHOW PARTITIONS`
    with a spec on the second column, and drops, re-adds and drops the null
    partition written by an `INSERT`.
---
 .../utils/HiveGravitinoOperationOperator.java      | 44 +++++------
 .../spark/connector/utils/SparkPartitionUtils.java |  2 +-
 .../integration/test/hive/SparkHiveCatalogIT.java  | 13 ++++
 .../utils/TestHiveGravitinoOperationOperator.java  | 91 ++++++++++++++++++++++
 4 files changed, 125 insertions(+), 25 deletions(-)

diff --git 
a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/utils/HiveGravitinoOperationOperator.java
 
b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/utils/HiveGravitinoOperationOperator.java
index a726bc8887..ec3eadfbb1 100644
--- 
a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/utils/HiveGravitinoOperationOperator.java
+++ 
b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/utils/HiveGravitinoOperationOperator.java
@@ -27,6 +27,7 @@ import java.util.List;
 import java.util.Map;
 import org.apache.gravitino.exceptions.NoSuchPartitionException;
 import org.apache.gravitino.rel.expressions.literals.Literal;
+import org.apache.gravitino.rel.expressions.literals.Literals;
 import org.apache.gravitino.rel.partitions.Partition;
 import org.apache.gravitino.rel.partitions.Partitions;
 import org.apache.spark.sql.catalyst.InternalRow;
@@ -64,7 +65,11 @@ public class HiveGravitinoOperationOperator {
       StructField structField = partitionSchema.apply(i);
       DataType dataType = structField.dataType();
       fields.add(new String[] {structField.name()});
-      values.add(SparkPartitionUtils.toGravitinoLiteral(ident, i, dataType));
+      // Hive and Spark write a null partition value as 
__HIVE_DEFAULT_PARTITION__.
+      values.add(
+          ident.isNullAt(i)
+              ? 
Literals.stringLiteral(SparkPartitionUtils.HIVE_DEFAULT_PARTITION_NAME)
+              : SparkPartitionUtils.toGravitinoLiteral(ident, i, dataType));
     }
 
     Partition partition =
@@ -95,29 +100,16 @@ public class HiveGravitinoOperationOperator {
 
     boolean isNeedAll = names != null && names.length == 0 ? true : false;
 
-    String[] partitionNames =
-        getHivePartitionName(names, ident, 
partitionSchema).split(PARTITION_NAME_DELIMITER);
-    String[] partitionNamesWithDelimiter =
-        Arrays.stream(partitionNames)
-            .map(name -> name + PARTITION_NAME_DELIMITER)
-            .toArray(String[]::new);
+    List<String> requestedParts =
+        Arrays.asList(
+            getHivePartitionName(names, ident, 
partitionSchema).split(PARTITION_NAME_DELIMITER));
 
     return Arrays.stream(allPartitions)
-        .map(e -> e + PARTITION_NAME_DELIMITER)
+        // Match whole column=value parts; a substring match lets region=us 
match subregion=us.
         .filter(
-            e -> {
-              if (isNeedAll) {
-                return true;
-              }
-              for (String name : partitionNamesWithDelimiter) {
-                // exactly match
-                if (!e.contains(name)) {
-                  return false;
-                }
-              }
-              return true;
-            })
-        .map(e -> e.substring(0, e.length() - 1))
+            e ->
+                isNeedAll
+                    || 
Arrays.asList(e.split(PARTITION_NAME_DELIMITER)).containsAll(requestedParts))
         .map(e -> toSparkPartition(e, partitionSchema))
         .toArray(GenericInternalRow[]::new);
   }
@@ -159,12 +151,15 @@ public class HiveGravitinoOperationOperator {
       String[] names, InternalRow ident, StructType partitionSchema) {
     StringBuilder partitionName = new StringBuilder();
     Preconditions.checkArgument(names != null, "Partition column names must 
not be null");
+    // names can be any subset of the partition columns, so look each column 
up by name.
     for (int i = 0; i < names.length; i++) {
-      StructField structField = partitionSchema.apply(i);
+      StructField structField = partitionSchema.apply(names[i]);
       DataType dataType = structField.dataType();
+      String value = SparkPartitionUtils.getPartitionValueAsString(ident, i, 
dataType);
+      // Hive and Spark write a null partition value as 
__HIVE_DEFAULT_PARTITION__.
       partitionName.append(
           partitionNamePart(
-              names[i], SparkPartitionUtils.getPartitionValueAsString(ident, 
i, dataType)));
+              names[i], value == null ? 
SparkPartitionUtils.HIVE_DEFAULT_PARTITION_NAME : value));
       if (i < names.length - 1) {
         partitionName.append(PARTITION_NAME_DELIMITER);
       }
@@ -178,10 +173,11 @@ public class HiveGravitinoOperationOperator {
     for (int i = 0; i < numFields; i++) {
       StructField structField = partitionSchema.apply(i);
       DataType dataType = structField.dataType();
+      String value = SparkPartitionUtils.getPartitionValueAsString(ident, i, 
dataType);
       partitionName.append(
           partitionNamePart(
               structField.name(),
-              SparkPartitionUtils.getPartitionValueAsString(ident, i, 
dataType)));
+              value == null ? SparkPartitionUtils.HIVE_DEFAULT_PARTITION_NAME 
: value));
       if (i < numFields - 1) {
         partitionName.append(PARTITION_NAME_DELIMITER);
       }
diff --git 
a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/utils/SparkPartitionUtils.java
 
b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/utils/SparkPartitionUtils.java
index 4d2f574d19..f058333ebf 100644
--- 
a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/utils/SparkPartitionUtils.java
+++ 
b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/utils/SparkPartitionUtils.java
@@ -43,7 +43,7 @@ import org.apache.spark.unsafe.types.UTF8String;
 
 public class SparkPartitionUtils {
 
-  private static final String HIVE_DEFAULT_PARTITION_NAME = 
"__HIVE_DEFAULT_PARTITION__";
+  static final String HIVE_DEFAULT_PARTITION_NAME = 
"__HIVE_DEFAULT_PARTITION__";
   private static final String HIVE_DEFAULT_PARTITION_NAME_LEGACY = 
"HIVE_DEFAULT_PARTITION";
 
   private SparkPartitionUtils() {}
diff --git 
a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/hive/SparkHiveCatalogIT.java
 
b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/hive/SparkHiveCatalogIT.java
index 83cdcfd6fc..9e9ab3b44d 100644
--- 
a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/hive/SparkHiveCatalogIT.java
+++ 
b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/hive/SparkHiveCatalogIT.java
@@ -191,6 +191,11 @@ public abstract class SparkHiveCatalogIT extends 
SparkCommonIT {
     partitionInfo = sql("SHOW PARTITIONS " + tableName + " PARTITION 
(age_p1=2)");
     Assertions.assertEquals(0, partitionInfo.size());
 
+    // A partial spec on the second partition column.
+    partitionInfo = sql("SHOW PARTITIONS " + tableName + " PARTITION 
(age_p2='twenty two')");
+    Assertions.assertEquals(1, partitionInfo.size());
+    Assertions.assertEquals("age_p1=22/age_p2=twenty two", 
partitionInfo.get(0)[0]);
+
     String nullPartitionTableName = "hive_null_partition_ops_table";
     dropTableIfExists(nullPartitionTableName);
     sql(
@@ -202,6 +207,14 @@ public abstract class SparkHiveCatalogIT extends 
SparkCommonIT {
     Assertions.assertEquals(1, nullPartitionInfo.size());
     
Assertions.assertTrue(String.valueOf(nullPartitionInfo.get(0)[0]).startsWith("dt="));
 
+    // The null partition written by the INSERT can be dropped, and re-added 
through the connector.
+    sql("ALTER TABLE " + nullPartitionTableName + " DROP PARTITION (dt = 
null)");
+    Assertions.assertEquals(0, 
getTablePartitions(nullPartitionTableName).size());
+    sql("ALTER TABLE " + nullPartitionTableName + " ADD PARTITION (dt = 
null)");
+    Assertions.assertEquals(1, 
getTablePartitions(nullPartitionTableName).size());
+    sql("ALTER TABLE " + nullPartitionTableName + " DROP PARTITION (dt = 
null)");
+    Assertions.assertEquals(0, 
getTablePartitions(nullPartitionTableName).size());
+
     // Adding a partition that already exists must surface Spark's 
PARTITIONS_ALREADY_EXIST. Assert
     // the type rather than the message: the connector builds the exception 
from (table, ident,
     // schema), the only constructor Spark 3.5 and 4.x share, so the wording 
is Spark's own.
diff --git 
a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/utils/TestHiveGravitinoOperationOperator.java
 
b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/utils/TestHiveGravitinoOperationOperator.java
index b207d309c8..f8f7371b66 100644
--- 
a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/utils/TestHiveGravitinoOperationOperator.java
+++ 
b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/utils/TestHiveGravitinoOperationOperator.java
@@ -23,8 +23,12 @@ import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
+import java.util.Collections;
 import org.apache.gravitino.rel.SupportsPartitions;
 import org.apache.gravitino.rel.Table;
+import org.apache.gravitino.rel.expressions.literals.Literal;
+import org.apache.gravitino.rel.partitions.IdentityPartition;
+import org.apache.gravitino.rel.partitions.Partition;
 import org.apache.spark.sql.catalyst.InternalRow;
 import org.apache.spark.sql.catalyst.expressions.GenericInternalRow;
 import org.apache.spark.sql.types.DataTypes;
@@ -33,12 +37,18 @@ import org.apache.spark.unsafe.types.UTF8String;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
 
 public class TestHiveGravitinoOperationOperator {
 
   private static final StructType PARTITION_SCHEMA =
       new StructType().add("ds", DataTypes.StringType);
 
+  private static final StructType MULTI_KEY_SCHEMA =
+      new StructType()
+          .add("dt", DataTypes.IntegerType, true)
+          .add("region", DataTypes.StringType, true);
+
   private SupportsPartitions partitions;
   private HiveGravitinoOperationOperator operator;
 
@@ -69,6 +79,87 @@ public class TestHiveGravitinoOperationOperator {
     Assertions.assertEquals("a/b", rows[0].getUTF8String(0).toString());
   }
 
+  @Test
+  void testNullPartitionValueUsesHiveDefaultMarker() {
+    ArgumentCaptor<String> name = ArgumentCaptor.forClass(String.class);
+    when(partitions.dropPartition(name.capture())).thenReturn(true);
+
+    InternalRow ident = new GenericInternalRow(new Object[] {20240101, null});
+    operator.dropPartition(ident, MULTI_KEY_SCHEMA);
+
+    Assertions.assertEquals("dt=20240101/region=__HIVE_DEFAULT_PARTITION__", 
name.getValue());
+  }
+
+  @Test
+  void testAddNullPartitionValueUsesHiveDefaultMarker() throws Exception {
+    ArgumentCaptor<Partition> partition = 
ArgumentCaptor.forClass(Partition.class);
+    when(partitions.addPartition(partition.capture())).thenReturn(null);
+
+    InternalRow ident = new GenericInternalRow(new Object[] {20240101, null});
+    operator.createPartition(ident, Collections.emptyMap(), MULTI_KEY_SCHEMA);
+
+    Literal<?>[] values = ((IdentityPartition) partition.getValue()).values();
+    Assertions.assertEquals(20240101, values[0].value());
+    Assertions.assertEquals("__HIVE_DEFAULT_PARTITION__", values[1].value());
+  }
+
+  @Test
+  void testPartialSpecResolvesColumnsByName() {
+    when(partitions.listPartitionNames())
+        .thenReturn(new String[] {"dt=1/region=us", "dt=2/region=eu"});
+
+    // SHOW PARTITIONS ... PARTITION (region = 'us') passes only the region 
column and its value.
+    InternalRow ident = new GenericInternalRow(new Object[] 
{UTF8String.fromString("us")});
+    InternalRow[] result =
+        operator.listPartitionIdentifiers(new String[] {"region"}, ident, 
MULTI_KEY_SCHEMA);
+
+    Assertions.assertEquals(1, result.length);
+    Assertions.assertEquals(1, result[0].getInt(0));
+    Assertions.assertEquals("us", result[0].getUTF8String(1).toString());
+  }
+
+  @Test
+  void testNullValueInNonStringColumnUsesHiveDefaultMarker() throws Exception {
+    InternalRow ident = new GenericInternalRow(new Object[] {null, 
UTF8String.fromString("us")});
+
+    ArgumentCaptor<Partition> partition = 
ArgumentCaptor.forClass(Partition.class);
+    when(partitions.addPartition(partition.capture())).thenReturn(null);
+    operator.createPartition(ident, Collections.emptyMap(), MULTI_KEY_SCHEMA);
+    Literal<?>[] values = ((IdentityPartition) partition.getValue()).values();
+    Assertions.assertEquals("__HIVE_DEFAULT_PARTITION__", values[0].value());
+
+    ArgumentCaptor<String> name = ArgumentCaptor.forClass(String.class);
+    when(partitions.getPartition(name.capture())).thenReturn(null);
+    operator.loadPartitionMetadata(ident, MULTI_KEY_SCHEMA);
+    Assertions.assertEquals("dt=__HIVE_DEFAULT_PARTITION__/region=us", 
name.getValue());
+  }
+
+  @Test
+  void testPartitionExistsWithNullInPartialSpec() {
+    
when(partitions.partitionExists("region=__HIVE_DEFAULT_PARTITION__")).thenReturn(true);
+
+    InternalRow ident = new GenericInternalRow(new Object[] {null});
+    Assertions.assertTrue(
+        operator.partitionExists(new String[] {"region"}, ident, 
MULTI_KEY_SCHEMA));
+  }
+
+  @Test
+  void testPartialSpecMatchesWholeColumnName() {
+    StructType schema =
+        new StructType()
+            .add("region", DataTypes.StringType, true)
+            .add("subregion", DataTypes.StringType, true);
+    when(partitions.listPartitionNames())
+        .thenReturn(new String[] {"region=eu/subregion=us", 
"region=us/subregion=eu"});
+
+    InternalRow[] result =
+        operator.listPartitionIdentifiers(new String[] {"region"}, 
ident("us"), schema);
+
+    Assertions.assertEquals(1, result.length);
+    Assertions.assertEquals("us", result[0].getUTF8String(0).toString());
+    Assertions.assertEquals("eu", result[0].getUTF8String(1).toString());
+  }
+
   private static InternalRow ident(String value) {
     return new GenericInternalRow(new Object[] {UTF8String.fromString(value)});
   }

Reply via email to