This is an automated email from the ASF dual-hosted git repository.
jerryshao pushed a commit to branch branch-1.3
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/branch-1.3 by this push:
new c1976e1f58 [Cherry-pick to branch-1.3] [#14685] fix(hive): reject
unsupported distribution strategies (#14686) (#14701)
c1976e1f58 is described below
commit c1976e1f5843fb2f32cef162c836271431572d12
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Fri Nov 6 17:44:47 2026 +0800
[Cherry-pick to branch-1.3] [#14685] fix(hive): reject unsupported
distribution strategies (#14686) (#14701)
**Cherry-pick Information:**
- Original commit: 2d30655f173920c6d6ed1da2e9573149b7c7184d
- Target branch: `branch-1.3`
- Status: ✅ Clean cherry-pick (no conflicts)
Co-authored-by: Qi Yu <[email protected]>
---
.../catalog/hive/HiveCatalogOperations.java | 9 +++
.../catalog/hive/TestHiveCatalogOperations.java | 77 ++++++++++++++++++++++
.../hive/integration/test/CatalogHive2IT.java | 23 ++++++-
.../client/integration/test/StatisticIT.java | 2 +-
docs/apache-hive-catalog.md | 2 +-
5 files changed, 108 insertions(+), 5 deletions(-)
diff --git
a/catalogs/catalog-hive/src/main/java/org/apache/gravitino/catalog/hive/HiveCatalogOperations.java
b/catalogs/catalog-hive/src/main/java/org/apache/gravitino/catalog/hive/HiveCatalogOperations.java
index 653dbbf050..f69b1b584e 100644
---
a/catalogs/catalog-hive/src/main/java/org/apache/gravitino/catalog/hive/HiveCatalogOperations.java
+++
b/catalogs/catalog-hive/src/main/java/org/apache/gravitino/catalog/hive/HiveCatalogOperations.java
@@ -85,6 +85,7 @@ import org.apache.gravitino.rel.expressions.Expression;
import org.apache.gravitino.rel.expressions.NamedReference;
import org.apache.gravitino.rel.expressions.distributions.Distribution;
import org.apache.gravitino.rel.expressions.distributions.Distributions;
+import org.apache.gravitino.rel.expressions.distributions.Strategy;
import org.apache.gravitino.rel.expressions.sorts.SortOrder;
import org.apache.gravitino.rel.expressions.transforms.Transform;
import org.apache.gravitino.rel.expressions.transforms.Transforms;
@@ -584,6 +585,14 @@ public class HiveCatalogOperations
private void validateDistributionAndSort(Distribution distribution,
SortOrder[] sortOrder) {
if (distribution != Distributions.NONE) {
+ // NONE can also arrive as a DTO rather than the singleton used by the
Java API.
+ boolean unbucketed =
+ distribution.strategy() == Strategy.NONE
+ && distribution.number() == 0
+ && distribution.expressions().length == 0;
+ Preconditions.checkArgument(
+ distribution.strategy() == Strategy.HASH || unbucketed,
+ "Hive only supports HASH distribution, or NONE with zero buckets and
no expressions");
boolean allNameReference =
Arrays.stream(distribution.expressions())
.allMatch(t -> t instanceof NamedReference.FieldReference);
diff --git
a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/TestHiveCatalogOperations.java
b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/TestHiveCatalogOperations.java
index b93e9c8c80..0a03bbd1dc 100644
---
a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/TestHiveCatalogOperations.java
+++
b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/TestHiveCatalogOperations.java
@@ -46,6 +46,7 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
import com.google.common.collect.ImmutableMap;
@@ -60,6 +61,7 @@ import org.apache.gravitino.NameIdentifier;
import org.apache.gravitino.Namespace;
import org.apache.gravitino.connector.BaseCatalog;
import org.apache.gravitino.connector.PropertyEntry;
+import org.apache.gravitino.dto.rel.DistributionDTO;
import org.apache.gravitino.exceptions.ConnectionFailedException;
import org.apache.gravitino.exceptions.ViewAlreadyExistsException;
import org.apache.gravitino.hive.CachedClientPool;
@@ -73,7 +75,9 @@ import org.apache.gravitino.rel.SQLRepresentation;
import org.apache.gravitino.rel.TableChange;
import org.apache.gravitino.rel.View;
import org.apache.gravitino.rel.ViewChange;
+import org.apache.gravitino.rel.expressions.distributions.Distribution;
import org.apache.gravitino.rel.expressions.distributions.Distributions;
+import org.apache.gravitino.rel.expressions.distributions.Strategy;
import org.apache.gravitino.rel.expressions.sorts.SortOrder;
import org.apache.gravitino.rel.expressions.transforms.Transform;
import org.apache.gravitino.rel.indexes.Index;
@@ -222,6 +226,79 @@ class TestHiveCatalogOperations {
exception.getMessage());
}
+ @Test
+ void testRejectNonHashDistributionBeforeMetastoreAccess() throws Exception {
+ for (Strategy strategy : new Strategy[] {Strategy.RANGE, Strategy.EVEN,
Strategy.NONE}) {
+ try (HiveCatalogOperations op = new HiveCatalogOperations()) {
+ op.initialize(Maps.newHashMap(), null, HIVE_PROPERTIES_METADATA);
+ CachedClientPool pool = mock(CachedClientPool.class);
+ HiveClient client = mock(HiveClient.class);
+ when(client.getDatabase(anyString(), anyString()))
+
.thenReturn(HiveSchema.builder().withCatalogName("hive").withName("db").build());
+ when(pool.run(any()))
+ .thenAnswer(
+ invocation -> {
+ ClientPool.Action<?, HiveClient, ?> action =
invocation.getArgument(0);
+ return action.run(client);
+ });
+ op.clientPool = pool;
+ IllegalArgumentException error =
+ Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () ->
+ op.createTable(
+ NameIdentifier.of("db", "invalid_distribution"),
+ new Column[] {Column.of("id", Types.IntegerType.get(),
null)},
+ null,
+ Map.of("is_generic", "true"),
+ new Transform[0],
+ Distributions.fields(strategy, 4, new String[] {"id"}),
+ new SortOrder[0],
+ new Index[0]));
+ Assertions.assertTrue(error.getMessage().contains("HASH"));
+ verifyNoInteractions(pool, client);
+ }
+ }
+ }
+
+ @Test
+ void testHashAndStructuralNoneDistributionsRemainSupported() throws
Exception {
+ try (HiveCatalogOperations op = new HiveCatalogOperations()) {
+ op.initialize(Maps.newHashMap(), null, HIVE_PROPERTIES_METADATA);
+ CachedClientPool pool = mock(CachedClientPool.class);
+ HiveClient client = mock(HiveClient.class);
+ when(client.getDatabase(anyString(), anyString()))
+
.thenReturn(HiveSchema.builder().withCatalogName("hive").withName("db").build());
+ when(pool.run(any()))
+ .thenAnswer(
+ invocation -> {
+ ClientPool.Action<?, HiveClient, ?> action =
invocation.getArgument(0);
+ return action.run(client);
+ });
+ op.clientPool = pool;
+ for (Distribution distribution :
+ new Distribution[] {
+ Distributions.fields(Strategy.HASH, 4, new String[] {"id"}),
+ Distributions.NONE,
+ Distributions.of(Strategy.NONE, 0),
+ DistributionDTO.NONE
+ }) {
+ Assertions.assertDoesNotThrow(
+ () ->
+ op.createTable(
+ NameIdentifier.of("db", "supported_distribution"),
+ new Column[] {Column.of("id", Types.IntegerType.get(),
null)},
+ null,
+ Map.of("is_generic", "true"),
+ new Transform[0],
+ distribution,
+ new SortOrder[0],
+ new Index[0]));
+ }
+ verify(client, times(4)).createTable(any());
+ }
+ }
+
@Test
void testCreateGenericTableWithEmptyColumns() throws Exception {
HiveCatalogOperations op = new HiveCatalogOperations();
diff --git
a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHive2IT.java
b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHive2IT.java
index 6798cc79e2..8f609c20b9 100644
---
a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHive2IT.java
+++
b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHive2IT.java
@@ -561,6 +561,7 @@ public class CatalogHive2IT extends BaseIT {
return;
}
+ Assertions.assertEquals(expected.strategy(), actual.strategy());
Assertions.assertEquals(expected.number(), actual.number());
List<String> expectedFields =
Arrays.stream(expected.expressions())
@@ -610,7 +611,7 @@ public class CatalogHive2IT extends BaseIT {
NameIdentifier nameIdentifier = NameIdentifier.of(schemaName, tableName);
Distribution distribution =
- Distributions.of(Strategy.EVEN, 10,
NamedReference.field(HIVE_COL_NAME1));
+ Distributions.of(Strategy.HASH, 10,
NamedReference.field(HIVE_COL_NAME1));
final SortOrder[] sortOrders =
new SortOrder[] {
@@ -618,6 +619,23 @@ public class CatalogHive2IT extends BaseIT {
};
Map<String, String> properties = createProperties();
+ TableCatalog tableCatalog = catalog.asTableCatalog();
+ for (Strategy unsupported : new Strategy[] {Strategy.RANGE,
Strategy.EVEN}) {
+ IllegalArgumentException error =
+ Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () ->
+ tableCatalog.createTable(
+ nameIdentifier,
+ columns,
+ TABLE_COMMENT,
+ properties,
+ Transforms.EMPTY_TRANSFORM,
+ Distributions.of(unsupported, 10,
NamedReference.field(HIVE_COL_NAME1)),
+ sortOrders));
+ Assertions.assertTrue(error.getMessage().contains("HASH"));
+ Assertions.assertFalse(tableCatalog.tableExists(nameIdentifier));
+ }
Table createdTable =
catalog
.asTableCatalog()
@@ -658,8 +676,7 @@ public class CatalogHive2IT extends BaseIT {
// Test bad request
// Bad name in distribution
final Distribution badDistribution =
- Distributions.of(Strategy.EVEN, 10,
NamedReference.field(HIVE_COL_NAME1 + "bad_name"));
- TableCatalog tableCatalog = catalog.asTableCatalog();
+ Distributions.of(Strategy.HASH, 10,
NamedReference.field(HIVE_COL_NAME1 + "bad_name"));
Assertions.assertThrows(
Exception.class,
() -> {
diff --git
a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/StatisticIT.java
b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/StatisticIT.java
index e3d3b255b1..f80bf2a4b5 100644
---
a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/StatisticIT.java
+++
b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/StatisticIT.java
@@ -325,7 +325,7 @@ public class StatisticIT extends BaseIT {
NameIdentifier nameIdentifier = NameIdentifier.of(schemaName, tableName);
Distribution distribution =
- Distributions.of(Strategy.EVEN, 10,
NamedReference.field(HIVE_COL_NAME1));
+ Distributions.of(Strategy.HASH, 10,
NamedReference.field(HIVE_COL_NAME1));
final SortOrder[] sortOrders =
new SortOrder[] {
diff --git a/docs/apache-hive-catalog.md b/docs/apache-hive-catalog.md
index 3266b3fc24..1ff786ed54 100644
--- a/docs/apache-hive-catalog.md
+++ b/docs/apache-hive-catalog.md
@@ -124,7 +124,7 @@ The `fieldName` specified in the partitioning attribute
must be the name of a co
### Table Sort Orders and Distributions
The Hive catalog supports [bucketed sorted
tables](https://cwiki.apache.org/confluence/display/Hive/LanguageManual+DDL#LanguageManualDDL-BucketedSortedTables).
Users can create bucketed sorted tables in the Hive catalog with specific
`distribution` and `sortOrders` attributes.
-Although Gravitino supports several distribution strategies, Apache Hive
inherently only supports a single distribution strategy (clustered by column).
Therefore the Hive catalog only supports `Hash` distribution.
+Although Gravitino supports several distribution strategies, Apache Hive
inherently only supports a single distribution strategy (clustered by column).
Therefore the Hive catalog only supports `Hash` bucketing. Use `none` (zero
buckets and no expressions) for an unbucketed table. Other distribution
strategies are rejected rather than silently converted to hash bucketing.
:::caution
The `fieldName` specified in the `distribution` and `sortOrders` attribute
must be the name of a column defined in the table.