This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new d8f9beade6 [hive] Add catalog option to control alter table cascade
(#8210)
d8f9beade6 is described below
commit d8f9beade6f480212015c38ff1c81fffb19f66e7
Author: Liurnly <[email protected]>
AuthorDate: Fri Jun 12 15:10:34 2026 +0800
[hive] Add catalog option to control alter table cascade (#8210)
When Paimon alters a Hive table, it unconditionally sets
`StatsSetupConst.CASCADE = "true"` in the `EnvironmentContext`, which
makes Hive cascade the schema change to every partition. On large
partitioned tables this can be expensive, and there is currently no way
to turn it off.
This PR adds a new catalog option `alter-table-cascade` (default `true`,
preserving the current behavior) to control whether the cascade is
applied. Set `alter-table-cascade=false` to skip cascading schema
changes to partitions.
---
docs/generated/hive_catalog_configuration.html | 6 +
.../apache/paimon/hive/HiveAlterTableUtils.java | 25 +++-
.../java/org/apache/paimon/hive/HiveCatalog.java | 7 +-
.../org/apache/paimon/hive/HiveCatalogOptions.java | 7 +
.../paimon/hive/HiveAlterTableUtilsTest.java | 144 +++++++++++++++++++++
5 files changed, 181 insertions(+), 8 deletions(-)
diff --git a/docs/generated/hive_catalog_configuration.html
b/docs/generated/hive_catalog_configuration.html
index 63eaab9327..0ce809098a 100644
--- a/docs/generated/hive_catalog_configuration.html
+++ b/docs/generated/hive_catalog_configuration.html
@@ -26,6 +26,12 @@ under the License.
</tr>
</thead>
<tbody>
+ <tr>
+ <td><h5>alter-table-cascade</h5></td>
+ <td style="word-wrap: break-word;">true</td>
+ <td>Boolean</td>
+ <td>Whether to cascade schema changes to Hive metastore partitions
when altering table.</td>
+ </tr>
<tr>
<td><h5>client-pool-cache.eviction-interval-ms</h5></td>
<td style="word-wrap: break-word;">300000</td>
diff --git
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveAlterTableUtils.java
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveAlterTableUtils.java
index 7759a3fd93..4e183c5812 100644
---
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveAlterTableUtils.java
+++
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveAlterTableUtils.java
@@ -30,23 +30,33 @@ import org.apache.thrift.TException;
public class HiveAlterTableUtils {
public static void alterTable(
- IMetaStoreClient client, Identifier identifier, Table table,
boolean skipUpdateStats)
+ IMetaStoreClient client,
+ Identifier identifier,
+ Table table,
+ boolean skipUpdateStats,
+ boolean cascade)
throws TException {
try {
- alterTableWithEnv(client, identifier, table, skipUpdateStats);
+ alterTableWithEnv(client, identifier, table, skipUpdateStats,
cascade);
} catch (NoClassDefFoundError | NoSuchMethodError e) {
- alterTableWithoutEnv(client, identifier, table);
+ alterTableWithoutEnv(client, identifier, table, cascade);
}
}
private static void alterTableWithEnv(
- IMetaStoreClient client, Identifier identifier, Table table,
boolean skipUpdateStats)
+ IMetaStoreClient client,
+ Identifier identifier,
+ Table table,
+ boolean skipUpdateStats,
+ boolean cascade)
throws TException {
boolean skipHiveUpdateStats =
Boolean.parseBoolean(table.getParameters().get(StatsSetupConst.DO_NOT_UPDATE_STATS))
|| skipUpdateStats;
EnvironmentContext environmentContext = new EnvironmentContext();
- environmentContext.putToProperties(StatsSetupConst.CASCADE, "true");
+ if (cascade) {
+ environmentContext.putToProperties(StatsSetupConst.CASCADE,
"true");
+ }
environmentContext.putToProperties(
StatsSetupConst.DO_NOT_UPDATE_STATS,
Boolean.toString(skipHiveUpdateStats));
client.alter_table_with_environmentContext(
@@ -54,7 +64,8 @@ public class HiveAlterTableUtils {
}
private static void alterTableWithoutEnv(
- IMetaStoreClient client, Identifier identifier, Table table)
throws TException {
- client.alter_table(identifier.getDatabaseName(),
identifier.getTableName(), table, true);
+ IMetaStoreClient client, Identifier identifier, Table table,
boolean cascade)
+ throws TException {
+ client.alter_table(identifier.getDatabaseName(),
identifier.getTableName(), table, cascade);
}
}
diff --git
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
index c988d2b41a..889850bf11 100644
---
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
+++
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
@@ -114,6 +114,7 @@ import static
org.apache.paimon.catalog.CatalogUtils.isSystemDatabase;
import static
org.apache.paimon.catalog.CatalogUtils.listPartitionsFromFileSystem;
import static org.apache.paimon.catalog.Identifier.DEFAULT_MAIN_BRANCH;
import static org.apache.paimon.format.csv.CsvOptions.FIELD_DELIMITER;
+import static org.apache.paimon.hive.HiveCatalogOptions.ALTER_TABLE_CASCADE;
import static org.apache.paimon.hive.HiveCatalogOptions.HADOOP_CONF_DIR;
import static org.apache.paimon.hive.HiveCatalogOptions.HIVE_CONF_DIR;
import static org.apache.paimon.hive.HiveCatalogOptions.HIVE_SKIP_UPDATE_STATS;
@@ -1298,7 +1299,11 @@ public class HiveCatalog extends AbstractCatalog {
.execute(
client ->
HiveAlterTableUtils.alterTable(
- client, identifier, table,
skipUpdateStats));
+ client,
+ identifier,
+ table,
+ skipUpdateStats,
+ options.get(ALTER_TABLE_CASCADE)));
}
@Override
diff --git
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogOptions.java
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogOptions.java
index 7700a763d0..9fc16b5946 100644
---
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogOptions.java
+++
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogOptions.java
@@ -102,5 +102,12 @@ public final class HiveCatalogOptions {
+ "E.g. specifying
\"conf:a.b.c\" will add \"a.b.c\" to the key, and so that configurations with
different default catalog wouldn't share the same client pool. Multiple conf
elements can be specified."))
.build());
+ public static final ConfigOption<Boolean> ALTER_TABLE_CASCADE =
+ ConfigOptions.key("alter-table-cascade")
+ .booleanType()
+ .defaultValue(true)
+ .withDescription(
+ "Whether to cascade schema changes to Hive
metastore partitions when altering table.");
+
private HiveCatalogOptions() {}
}
diff --git
a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveAlterTableUtilsTest.java
b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveAlterTableUtilsTest.java
new file mode 100644
index 0000000000..837499b7fa
--- /dev/null
+++
b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveAlterTableUtilsTest.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.hive;
+
+import org.apache.paimon.catalog.Identifier;
+
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.hadoop.hive.metastore.IMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.junit.jupiter.api.Test;
+
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+import java.util.HashMap;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Tests for {@link HiveAlterTableUtils}. */
+public class HiveAlterTableUtilsTest {
+
+ @Test
+ public void testAlterTableWithoutCascade() throws Exception {
+ CapturingMetaStoreClient handler = new CapturingMetaStoreClient(false);
+ IMetaStoreClient client = handler.client();
+ Identifier identifier = Identifier.create("db", "tbl");
+ Table table = new Table();
+ table.setParameters(new HashMap<>());
+
+ HiveAlterTableUtils.alterTable(client, identifier, table, true, false);
+
+ assertThat(handler.environmentContext.getProperties())
+ .containsEntry(StatsSetupConst.DO_NOT_UPDATE_STATS, "true")
+ .doesNotContainKey(StatsSetupConst.CASCADE);
+ }
+
+ @Test
+ public void testAlterTableWithCascade() throws Exception {
+ CapturingMetaStoreClient handler = new CapturingMetaStoreClient(false);
+ IMetaStoreClient client = handler.client();
+ Identifier identifier = Identifier.create("db", "tbl");
+ Table table = new Table();
+ table.setParameters(new HashMap<>());
+
+ HiveAlterTableUtils.alterTable(client, identifier, table, true, true);
+
+ assertThat(handler.environmentContext.getProperties())
+ .containsEntry(StatsSetupConst.DO_NOT_UPDATE_STATS, "true")
+ .containsEntry(StatsSetupConst.CASCADE, "true");
+ }
+
+ @Test
+ public void testAlterTableWithoutEnvUsesConfiguredCascade() throws
Exception {
+ CapturingMetaStoreClient handler = new CapturingMetaStoreClient(true);
+ IMetaStoreClient client = handler.client();
+ Identifier identifier = Identifier.create("db", "tbl");
+ Table table = new Table();
+ table.setParameters(new HashMap<>());
+
+ HiveAlterTableUtils.alterTable(client, identifier, table, true, false);
+
+ assertThat(handler.cascade).isFalse();
+ }
+
+ private static class CapturingMetaStoreClient implements InvocationHandler
{
+
+ private final boolean failWithEnv;
+ private EnvironmentContext environmentContext;
+ private Boolean cascade;
+
+ private CapturingMetaStoreClient(boolean failWithEnv) {
+ this.failWithEnv = failWithEnv;
+ }
+
+ private IMetaStoreClient client() {
+ return (IMetaStoreClient)
+ Proxy.newProxyInstance(
+ IMetaStoreClient.class.getClassLoader(),
+ new Class[] {IMetaStoreClient.class},
+ this);
+ }
+
+ @Override
+ public Object invoke(Object proxy, Method method, Object[] args) {
+ if
("alter_table_with_environmentContext".equals(method.getName())) {
+ if (failWithEnv) {
+ throw new NoSuchMethodError();
+ }
+ environmentContext = (EnvironmentContext) args[3];
+ return null;
+ }
+ if ("alter_table".equals(method.getName()) && args.length == 4) {
+ cascade = (Boolean) args[3];
+ return null;
+ }
+ return defaultValue(method.getReturnType());
+ }
+
+ private Object defaultValue(Class<?> returnType) {
+ if (!returnType.isPrimitive() || Void.TYPE.equals(returnType)) {
+ return null;
+ }
+ if (Boolean.TYPE.equals(returnType)) {
+ return false;
+ }
+ if (Character.TYPE.equals(returnType)) {
+ return '\0';
+ }
+ if (Byte.TYPE.equals(returnType)) {
+ return (byte) 0;
+ }
+ if (Short.TYPE.equals(returnType)) {
+ return (short) 0;
+ }
+ if (Integer.TYPE.equals(returnType)) {
+ return 0;
+ }
+ if (Long.TYPE.equals(returnType)) {
+ return 0L;
+ }
+ if (Float.TYPE.equals(returnType)) {
+ return 0F;
+ }
+ return 0D;
+ }
+ }
+}
