This is an automated email from the ASF dual-hosted git repository.
diqiu50 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 636288c60a [#10302] fix(spark-connector): handle Hive default
partition token in SHOW PARTITIONS (#11290)
636288c60a is described below
commit 636288c60a5b256724ddde742a7b911dd0b4b9c7
Author: mchades <[email protected]>
AuthorDate: Fri May 29 14:46:45 2026 +0800
[#10302] fix(spark-connector): handle Hive default partition token in SHOW
PARTITIONS (#11290)
### What changes were proposed in this pull request?
This PR fixes `SHOW PARTITIONS` failure in Spark connector when a Hive
partition value is null and represented as the default Hive partition
token for non-string partition columns.
Changes include:
- Treat `__HIVE_DEFAULT_PARTITION__` and `HIVE_DEFAULT_PARTITION` as
null in `SparkPartitionUtils.getSparkPartitionValue(...)`.
- Add unit assertions in `TestSparkPartitionUtils` to verify
token-to-null conversion.
- Extend Hive integration test coverage in
`SparkHiveCatalogIT.testManagePartitionTable` with a null `INT`
partition scenario for `SHOW PARTITIONS`.
### Why are the changes needed?
`SHOW PARTITIONS` could throw `UnsupportedOperationException` /
`NumberFormatException` by trying to parse Hive default partition tokens
(for null values) as numeric types.
Fix: #10302
### Does this PR introduce _any_ user-facing change?
Yes. `SHOW PARTITIONS` now works for Hive tables containing null values
in non-string partition columns.
### How was this patch tested?
- `./gradlew :spark-connector:spark-common:test --tests
org.apache.gravitino.spark.connector.utils.TestSparkPartitionUtils`
- `./gradlew :spark-connector:spark-3.5:test --tests
org.apache.gravitino.spark.connector.integration.test.hive.SparkHiveCatalogIT35.testManagePartitionTable
-PskipDockerTests=false`
- `./gradlew :spark-connector:spark-3.4:test --tests
org.apache.gravitino.spark.connector.integration.test.hive.SparkHiveCatalogIT34.testManagePartitionTable
-PskipDockerTests=false`
- `./gradlew :spark-connector:spark-3.3:test --tests
org.apache.gravitino.spark.connector.integration.test.hive.SparkHiveCatalogIT33.testManagePartitionTable
-PskipDockerTests=false`
---
.../gravitino/spark/connector/utils/SparkPartitionUtils.java | 10 +++++++++-
.../connector/integration/test/hive/SparkHiveCatalogIT.java | 11 +++++++++++
.../spark/connector/utils/TestSparkPartitionUtils.java | 10 ++++++++++
3 files changed, 30 insertions(+), 1 deletion(-)
diff --git
a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/utils/SparkPartitionUtils.java
b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/utils/SparkPartitionUtils.java
index 232c4a7254..21498f9af3 100644
---
a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/utils/SparkPartitionUtils.java
+++
b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/utils/SparkPartitionUtils.java
@@ -43,6 +43,9 @@ import org.apache.spark.unsafe.types.UTF8String;
public class SparkPartitionUtils {
+ private static final String HIVE_DEFAULT_PARTITION_NAME =
"__HIVE_DEFAULT_PARTITION__";
+ private static final String HIVE_DEFAULT_PARTITION_NAME_LEGACY =
"HIVE_DEFAULT_PARTITION";
+
private SparkPartitionUtils() {}
public static Literal<?> toGravitinoLiteral(InternalRow ident, int ordinal,
DataType sparkType) {
@@ -126,7 +129,7 @@ public class SparkPartitionUtils {
}
public static Object getSparkPartitionValue(String hivePartitionValue,
DataType dataType) {
- if (hivePartitionValue == null) {
+ if (hivePartitionValue == null ||
isHiveDefaultPartitionValue(hivePartitionValue)) {
return null;
}
try {
@@ -166,4 +169,9 @@ public class SparkPartitionUtils {
e);
}
}
+
+ private static boolean isHiveDefaultPartitionValue(String
hivePartitionValue) {
+ return HIVE_DEFAULT_PARTITION_NAME.equalsIgnoreCase(hivePartitionValue)
+ ||
HIVE_DEFAULT_PARTITION_NAME_LEGACY.equalsIgnoreCase(hivePartitionValue);
+ }
}
diff --git
a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/hive/SparkHiveCatalogIT.java
b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/hive/SparkHiveCatalogIT.java
index b0cd8716cb..c0ae42727d 100644
---
a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/hive/SparkHiveCatalogIT.java
+++
b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/hive/SparkHiveCatalogIT.java
@@ -166,6 +166,17 @@ public abstract class SparkHiveCatalogIT extends
SparkCommonIT {
partitionInfo = sql("SHOW PARTITIONS " + tableName + " PARTITION
(age_p1=2)");
Assertions.assertEquals(0, partitionInfo.size());
+ String nullPartitionTableName = "hive_null_partition_ops_table";
+ dropTableIfExists(nullPartitionTableName);
+ sql(
+ "CREATE TABLE "
+ + nullPartitionTableName
+ + " (id INT, name STRING) PARTITIONED BY (dt INT) STORED AS
PARQUET");
+ sql("INSERT INTO " + nullPartitionTableName + " (id, name, dt) SELECT 1,
'test', null");
+ List<Object[]> nullPartitionInfo =
getTablePartitions(nullPartitionTableName);
+ Assertions.assertEquals(1, nullPartitionInfo.size());
+
Assertions.assertTrue(String.valueOf(nullPartitionInfo.get(0)[0]).startsWith("dt="));
+
Exception exception =
Assertions.assertThrows(
Exception.class,
diff --git
a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/utils/TestSparkPartitionUtils.java
b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/utils/TestSparkPartitionUtils.java
index ea411e6ac1..19cf8c6059 100644
---
a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/utils/TestSparkPartitionUtils.java
+++
b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/utils/TestSparkPartitionUtils.java
@@ -157,6 +157,16 @@ public class TestSparkPartitionUtils {
SparkPartitionUtils.getSparkPartitionValue(hivePartitionValues[i],
dataType));
}
+ Assertions.assertNull(
+ SparkPartitionUtils.getSparkPartitionValue(
+ "__HIVE_DEFAULT_PARTITION__", DataTypes.IntegerType));
+ Assertions.assertNull(
+ SparkPartitionUtils.getSparkPartitionValue(
+ "HIVE_DEFAULT_PARTITION", DataTypes.IntegerType));
+ Assertions.assertNull(
+ SparkPartitionUtils.getSparkPartitionValue(
+ "__HIVE_DEFAULT_PARTITION__", DataTypes.StringType));
+
Assertions.assertThrowsExactly(
UnsupportedOperationException.class,
() ->
