This is an automated email from the ASF dual-hosted git repository.
jerryshao pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 68b3efb722 [#14291] fix(spark-connector): Use the Hive default
partition name for null values and resolve partial specs by name (#14295)
68b3efb722 is described below
commit 68b3efb7227d46e17acc810357033b11599788e9
Author: YangJie <[email protected]>
AuthorDate: Fri Nov 6 06:36:02 2026 -0500
[#14291] fix(spark-connector): Use the Hive default partition name for null
values and resolve partial specs by name (#14295)
### What changes were proposed in this pull request?
- `HiveGravitinoOperationOperator` writes a null partition value as
`__HIVE_DEFAULT_PARTITION__` in the partition names it builds for drop,
exists, load and list, and in the values it sends for `ADD PARTITION`.
The read side already maps this name back to null.
- The `names` variant of `getHivePartitionName` looks each column up by
name instead of by position, since Spark passes only the columns of a
partial spec.
- `listPartitionIdentifiers` matches whole `column=value` parts of each
partition name instead of substrings.
### Why are the changes needed?
Partitions with a null value could not be dropped or found, and `ADD
PARTITION` with a null value stored the string `null`. A partial spec on
any column but the first read the value with the wrong column's type,
which failed with a `ClassCastException` or matched nothing. The
substring match also let `region=us` match a partition with
`subregion=us`.
Fix: #14291
### Does this PR introduce _any_ user-facing change?
`DROP PARTITION (x = null)` and `ADD PARTITION (x = null)` now act on
the Hive default partition. Partitions added by an earlier connector
with `ADD PARTITION (x = null)` hold the string `null`; on a string
column they are now dropped with `x = 'null'`.
### How was this patch tested?
- `TestHiveGravitinoOperationOperator` adds cases for a null value on
drop, add, load and exists, in both an `INT` and a `STRING` column, for
a partial spec on the second column of an `INT, STRING` partitioned
table, and for a partial spec on `region` in a table that also has a
`subregion` column.
- `SparkHiveCatalogIT.testManagePartitionTable` adds `SHOW PARTITIONS`
with a spec on the second column, and drops, re-adds and drops the null
partition written by an `INSERT`.
---
.../utils/HiveGravitinoOperationOperator.java | 44 +++++------
.../spark/connector/utils/SparkPartitionUtils.java | 2 +-
.../integration/test/hive/SparkHiveCatalogIT.java | 13 ++++
.../utils/TestHiveGravitinoOperationOperator.java | 91 ++++++++++++++++++++++
4 files changed, 125 insertions(+), 25 deletions(-)
diff --git
a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/utils/HiveGravitinoOperationOperator.java
b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/utils/HiveGravitinoOperationOperator.java
index a726bc8887..ec3eadfbb1 100644
---
a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/utils/HiveGravitinoOperationOperator.java
+++
b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/utils/HiveGravitinoOperationOperator.java
@@ -27,6 +27,7 @@ import java.util.List;
import java.util.Map;
import org.apache.gravitino.exceptions.NoSuchPartitionException;
import org.apache.gravitino.rel.expressions.literals.Literal;
+import org.apache.gravitino.rel.expressions.literals.Literals;
import org.apache.gravitino.rel.partitions.Partition;
import org.apache.gravitino.rel.partitions.Partitions;
import org.apache.spark.sql.catalyst.InternalRow;
@@ -64,7 +65,11 @@ public class HiveGravitinoOperationOperator {
StructField structField = partitionSchema.apply(i);
DataType dataType = structField.dataType();
fields.add(new String[] {structField.name()});
- values.add(SparkPartitionUtils.toGravitinoLiteral(ident, i, dataType));
+ // Hive and Spark write a null partition value as
__HIVE_DEFAULT_PARTITION__.
+ values.add(
+ ident.isNullAt(i)
+ ?
Literals.stringLiteral(SparkPartitionUtils.HIVE_DEFAULT_PARTITION_NAME)
+ : SparkPartitionUtils.toGravitinoLiteral(ident, i, dataType));
}
Partition partition =
@@ -95,29 +100,16 @@ public class HiveGravitinoOperationOperator {
boolean isNeedAll = names != null && names.length == 0 ? true : false;
- String[] partitionNames =
- getHivePartitionName(names, ident,
partitionSchema).split(PARTITION_NAME_DELIMITER);
- String[] partitionNamesWithDelimiter =
- Arrays.stream(partitionNames)
- .map(name -> name + PARTITION_NAME_DELIMITER)
- .toArray(String[]::new);
+ List<String> requestedParts =
+ Arrays.asList(
+ getHivePartitionName(names, ident,
partitionSchema).split(PARTITION_NAME_DELIMITER));
return Arrays.stream(allPartitions)
- .map(e -> e + PARTITION_NAME_DELIMITER)
+ // Match whole column=value parts; a substring match lets region=us
match subregion=us.
.filter(
- e -> {
- if (isNeedAll) {
- return true;
- }
- for (String name : partitionNamesWithDelimiter) {
- // exactly match
- if (!e.contains(name)) {
- return false;
- }
- }
- return true;
- })
- .map(e -> e.substring(0, e.length() - 1))
+ e ->
+ isNeedAll
+ ||
Arrays.asList(e.split(PARTITION_NAME_DELIMITER)).containsAll(requestedParts))
.map(e -> toSparkPartition(e, partitionSchema))
.toArray(GenericInternalRow[]::new);
}
@@ -159,12 +151,15 @@ public class HiveGravitinoOperationOperator {
String[] names, InternalRow ident, StructType partitionSchema) {
StringBuilder partitionName = new StringBuilder();
Preconditions.checkArgument(names != null, "Partition column names must
not be null");
+ // names can be any subset of the partition columns, so look each column
up by name.
for (int i = 0; i < names.length; i++) {
- StructField structField = partitionSchema.apply(i);
+ StructField structField = partitionSchema.apply(names[i]);
DataType dataType = structField.dataType();
+ String value = SparkPartitionUtils.getPartitionValueAsString(ident, i,
dataType);
+ // Hive and Spark write a null partition value as
__HIVE_DEFAULT_PARTITION__.
partitionName.append(
partitionNamePart(
- names[i], SparkPartitionUtils.getPartitionValueAsString(ident,
i, dataType)));
+ names[i], value == null ?
SparkPartitionUtils.HIVE_DEFAULT_PARTITION_NAME : value));
if (i < names.length - 1) {
partitionName.append(PARTITION_NAME_DELIMITER);
}
@@ -178,10 +173,11 @@ public class HiveGravitinoOperationOperator {
for (int i = 0; i < numFields; i++) {
StructField structField = partitionSchema.apply(i);
DataType dataType = structField.dataType();
+ String value = SparkPartitionUtils.getPartitionValueAsString(ident, i,
dataType);
partitionName.append(
partitionNamePart(
structField.name(),
- SparkPartitionUtils.getPartitionValueAsString(ident, i,
dataType)));
+ value == null ? SparkPartitionUtils.HIVE_DEFAULT_PARTITION_NAME
: value));
if (i < numFields - 1) {
partitionName.append(PARTITION_NAME_DELIMITER);
}
diff --git
a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/utils/SparkPartitionUtils.java
b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/utils/SparkPartitionUtils.java
index 4d2f574d19..f058333ebf 100644
---
a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/utils/SparkPartitionUtils.java
+++
b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/utils/SparkPartitionUtils.java
@@ -43,7 +43,7 @@ import org.apache.spark.unsafe.types.UTF8String;
public class SparkPartitionUtils {
- private static final String HIVE_DEFAULT_PARTITION_NAME =
"__HIVE_DEFAULT_PARTITION__";
+ static final String HIVE_DEFAULT_PARTITION_NAME =
"__HIVE_DEFAULT_PARTITION__";
private static final String HIVE_DEFAULT_PARTITION_NAME_LEGACY =
"HIVE_DEFAULT_PARTITION";
private SparkPartitionUtils() {}
diff --git
a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/hive/SparkHiveCatalogIT.java
b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/hive/SparkHiveCatalogIT.java
index 83cdcfd6fc..9e9ab3b44d 100644
---
a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/hive/SparkHiveCatalogIT.java
+++
b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/hive/SparkHiveCatalogIT.java
@@ -191,6 +191,11 @@ public abstract class SparkHiveCatalogIT extends
SparkCommonIT {
partitionInfo = sql("SHOW PARTITIONS " + tableName + " PARTITION
(age_p1=2)");
Assertions.assertEquals(0, partitionInfo.size());
+ // A partial spec on the second partition column.
+ partitionInfo = sql("SHOW PARTITIONS " + tableName + " PARTITION
(age_p2='twenty two')");
+ Assertions.assertEquals(1, partitionInfo.size());
+ Assertions.assertEquals("age_p1=22/age_p2=twenty two",
partitionInfo.get(0)[0]);
+
String nullPartitionTableName = "hive_null_partition_ops_table";
dropTableIfExists(nullPartitionTableName);
sql(
@@ -202,6 +207,14 @@ public abstract class SparkHiveCatalogIT extends
SparkCommonIT {
Assertions.assertEquals(1, nullPartitionInfo.size());
Assertions.assertTrue(String.valueOf(nullPartitionInfo.get(0)[0]).startsWith("dt="));
+ // The null partition written by the INSERT can be dropped, and re-added
through the connector.
+ sql("ALTER TABLE " + nullPartitionTableName + " DROP PARTITION (dt =
null)");
+ Assertions.assertEquals(0,
getTablePartitions(nullPartitionTableName).size());
+ sql("ALTER TABLE " + nullPartitionTableName + " ADD PARTITION (dt =
null)");
+ Assertions.assertEquals(1,
getTablePartitions(nullPartitionTableName).size());
+ sql("ALTER TABLE " + nullPartitionTableName + " DROP PARTITION (dt =
null)");
+ Assertions.assertEquals(0,
getTablePartitions(nullPartitionTableName).size());
+
// Adding a partition that already exists must surface Spark's
PARTITIONS_ALREADY_EXIST. Assert
// the type rather than the message: the connector builds the exception
from (table, ident,
// schema), the only constructor Spark 3.5 and 4.x share, so the wording
is Spark's own.
diff --git
a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/utils/TestHiveGravitinoOperationOperator.java
b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/utils/TestHiveGravitinoOperationOperator.java
index b207d309c8..f8f7371b66 100644
---
a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/utils/TestHiveGravitinoOperationOperator.java
+++
b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/utils/TestHiveGravitinoOperationOperator.java
@@ -23,8 +23,12 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import java.util.Collections;
import org.apache.gravitino.rel.SupportsPartitions;
import org.apache.gravitino.rel.Table;
+import org.apache.gravitino.rel.expressions.literals.Literal;
+import org.apache.gravitino.rel.partitions.IdentityPartition;
+import org.apache.gravitino.rel.partitions.Partition;
import org.apache.spark.sql.catalyst.InternalRow;
import org.apache.spark.sql.catalyst.expressions.GenericInternalRow;
import org.apache.spark.sql.types.DataTypes;
@@ -33,12 +37,18 @@ import org.apache.spark.unsafe.types.UTF8String;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
public class TestHiveGravitinoOperationOperator {
private static final StructType PARTITION_SCHEMA =
new StructType().add("ds", DataTypes.StringType);
+ private static final StructType MULTI_KEY_SCHEMA =
+ new StructType()
+ .add("dt", DataTypes.IntegerType, true)
+ .add("region", DataTypes.StringType, true);
+
private SupportsPartitions partitions;
private HiveGravitinoOperationOperator operator;
@@ -69,6 +79,87 @@ public class TestHiveGravitinoOperationOperator {
Assertions.assertEquals("a/b", rows[0].getUTF8String(0).toString());
}
+ @Test
+ void testNullPartitionValueUsesHiveDefaultMarker() {
+ ArgumentCaptor<String> name = ArgumentCaptor.forClass(String.class);
+ when(partitions.dropPartition(name.capture())).thenReturn(true);
+
+ InternalRow ident = new GenericInternalRow(new Object[] {20240101, null});
+ operator.dropPartition(ident, MULTI_KEY_SCHEMA);
+
+ Assertions.assertEquals("dt=20240101/region=__HIVE_DEFAULT_PARTITION__",
name.getValue());
+ }
+
+ @Test
+ void testAddNullPartitionValueUsesHiveDefaultMarker() throws Exception {
+ ArgumentCaptor<Partition> partition =
ArgumentCaptor.forClass(Partition.class);
+ when(partitions.addPartition(partition.capture())).thenReturn(null);
+
+ InternalRow ident = new GenericInternalRow(new Object[] {20240101, null});
+ operator.createPartition(ident, Collections.emptyMap(), MULTI_KEY_SCHEMA);
+
+ Literal<?>[] values = ((IdentityPartition) partition.getValue()).values();
+ Assertions.assertEquals(20240101, values[0].value());
+ Assertions.assertEquals("__HIVE_DEFAULT_PARTITION__", values[1].value());
+ }
+
+ @Test
+ void testPartialSpecResolvesColumnsByName() {
+ when(partitions.listPartitionNames())
+ .thenReturn(new String[] {"dt=1/region=us", "dt=2/region=eu"});
+
+ // SHOW PARTITIONS ... PARTITION (region = 'us') passes only the region
column and its value.
+ InternalRow ident = new GenericInternalRow(new Object[]
{UTF8String.fromString("us")});
+ InternalRow[] result =
+ operator.listPartitionIdentifiers(new String[] {"region"}, ident,
MULTI_KEY_SCHEMA);
+
+ Assertions.assertEquals(1, result.length);
+ Assertions.assertEquals(1, result[0].getInt(0));
+ Assertions.assertEquals("us", result[0].getUTF8String(1).toString());
+ }
+
+ @Test
+ void testNullValueInNonStringColumnUsesHiveDefaultMarker() throws Exception {
+ InternalRow ident = new GenericInternalRow(new Object[] {null,
UTF8String.fromString("us")});
+
+ ArgumentCaptor<Partition> partition =
ArgumentCaptor.forClass(Partition.class);
+ when(partitions.addPartition(partition.capture())).thenReturn(null);
+ operator.createPartition(ident, Collections.emptyMap(), MULTI_KEY_SCHEMA);
+ Literal<?>[] values = ((IdentityPartition) partition.getValue()).values();
+ Assertions.assertEquals("__HIVE_DEFAULT_PARTITION__", values[0].value());
+
+ ArgumentCaptor<String> name = ArgumentCaptor.forClass(String.class);
+ when(partitions.getPartition(name.capture())).thenReturn(null);
+ operator.loadPartitionMetadata(ident, MULTI_KEY_SCHEMA);
+ Assertions.assertEquals("dt=__HIVE_DEFAULT_PARTITION__/region=us",
name.getValue());
+ }
+
+ @Test
+ void testPartitionExistsWithNullInPartialSpec() {
+
when(partitions.partitionExists("region=__HIVE_DEFAULT_PARTITION__")).thenReturn(true);
+
+ InternalRow ident = new GenericInternalRow(new Object[] {null});
+ Assertions.assertTrue(
+ operator.partitionExists(new String[] {"region"}, ident,
MULTI_KEY_SCHEMA));
+ }
+
+ @Test
+ void testPartialSpecMatchesWholeColumnName() {
+ StructType schema =
+ new StructType()
+ .add("region", DataTypes.StringType, true)
+ .add("subregion", DataTypes.StringType, true);
+ when(partitions.listPartitionNames())
+ .thenReturn(new String[] {"region=eu/subregion=us",
"region=us/subregion=eu"});
+
+ InternalRow[] result =
+ operator.listPartitionIdentifiers(new String[] {"region"},
ident("us"), schema);
+
+ Assertions.assertEquals(1, result.length);
+ Assertions.assertEquals("us", result[0].getUTF8String(0).toString());
+ Assertions.assertEquals("eu", result[0].getUTF8String(1).toString());
+ }
+
private static InternalRow ident(String value) {
return new GenericInternalRow(new Object[] {UTF8String.fromString(value)});
}