This is an automated email from the ASF dual-hosted git repository.

dengzhhu653 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
     new 807e5a8d362 HIVE-30238: Operations that support only direct SQL do 
nothing when m… (#6943)
807e5a8d362 is described below

commit 807e5a8d3628da3ebbd42c2bf4ce23a798a1742b
Author: dengzh <[email protected]>
AuthorDate: Thu Nov 5 06:05:38 2026 +0800

    HIVE-30238: Operations that support only direct SQL do nothing when m… 
(#6943)
---
 .../metastore/metastore/DirectSqlOnlyHelper.java   | 56 ++++++++++++++++
 .../hadoop/hive/metastore/metastore/GetHelper.java | 19 ++++--
 .../metastore/impl/ColStatsStoreImpl.java          | 76 ++++------------------
 .../metastore/metastore/impl/TableStoreImpl.java   | 25 ++-----
 .../hadoop/hive/metastore/TestObjectStore.java     | 48 ++++++++++++++
 5 files changed, 133 insertions(+), 91 deletions(-)

diff --git 
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/DirectSqlOnlyHelper.java
 
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/DirectSqlOnlyHelper.java
new file mode 100644
index 00000000000..ab0edd92226
--- /dev/null
+++ 
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/DirectSqlOnlyHelper.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.metastore;
+
+import java.util.List;
+
+import org.apache.hadoop.hive.metastore.api.MetaException;
+
+/**
+ * A {@link GetHelper} for operations that are only implemented with direct 
SQL, there is no JDO
+ * fallback. Such an operation runs with direct SQL regardless of 
metastore.try.direct.sql, and
+ * fails if direct SQL is not available on the datastore.
+ */
+public abstract class DirectSqlOnlyHelper<A, T> extends GetHelper<A, T> {
+
+  public DirectSqlOnlyHelper(RawStoreBundle rsb, A args) throws MetaException {
+    super(rsb, args);
+  }
+
+  public DirectSqlOnlyHelper(RawStoreBundle rsb, A args, List<String> fields) 
throws MetaException {
+    super(rsb, args, fields);
+  }
+
+  @Override
+  protected final boolean canUseDirectSql() {
+    return true;
+  }
+
+  @Override
+  protected final boolean canUseJdoQuery() {
+    return false;
+  }
+
+  @Override
+  protected final T getJdoResult() {
+    // Never called: GetHelper always takes the direct SQL path for these 
operations, or fails.
+    throw new UnsupportedOperationException("This operation is only supported 
by direct SQL");
+  }
+}
diff --git 
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/GetHelper.java
 
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/GetHelper.java
index eb921d1134f..e45f787cd64 100644
--- 
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/GetHelper.java
+++ 
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/GetHelper.java
@@ -76,16 +76,18 @@ public GetHelper(RawStoreBundle rsb,
     this.pm = rsb.getPersistentManager();
     this.allowJdo = canUseJdoQuery();
 
-    boolean isConfigEnabled = MetastoreConf.getBoolVar(baseStore.getConf(),
-        MetastoreConf.ConfVars.TRY_DIRECT_SQL);
-    if (isConfigEnabled) {
+    // metastore.try.direct.sql only chooses between direct SQL and JDO where 
both are available.
+    // An operation without a JDO path can only run with direct SQL, so it 
ignores the setting.
+    boolean tryDirectSql = !allowJdo ||
+        MetastoreConf.getBoolVar(baseStore.getConf(), 
MetastoreConf.ConfVars.TRY_DIRECT_SQL);
+    if (tryDirectSql) {
       directSql = new MetaStoreDirectSql(pm, baseStore.getConf(), "");
     }
 
-    if (!allowJdo && isConfigEnabled && !directSql.isCompatibleDatastore()) {
-      throw new MetaException("SQL is not operational"); // test path; SQL is 
enabled and broken.
+    if (!allowJdo && !directSql.isCompatibleDatastore()) {
+      throw new MetaException("This operation is only supported by direct SQL 
but the datastore is incompatible");
     }
-    this.doUseDirectSql = isConfigEnabled && directSql.isCompatibleDatastore();
+    this.doUseDirectSql = tryDirectSql && directSql.isCompatibleDatastore();
   }
 
   protected boolean canUseDirectSql() throws MetaException {
@@ -119,7 +121,10 @@ public T run(boolean initTable) throws MetaException, 
NoSuchObjectException {
       // Note that this will be invoked in 2 cases:
       //    1) DirectSQL was disabled to start with;
       //    2) DirectSQL threw and was disabled in handleDirectSqlError.
-      if (!doUseDirectSql && canUseJdoQuery()) {
+      if (!doUseDirectSql) {
+        if (!allowJdo) {
+          throw new MetaException("Cannot execute the direct SQL only 
operation as canUseDirectSql() got false");
+        }
         this.results = getJdoResult();
         LOG.debug("Not using direct SQL optimization.");
       }
diff --git 
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java
 
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java
index fcfb3c9d5a1..571fb6f2fc2 100644
--- 
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java
+++ 
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java
@@ -75,6 +75,7 @@
 import org.apache.hadoop.hive.metastore.directsql.DirectSqlAggrStats;
 import org.apache.hadoop.hive.metastore.directsql.DirectSqlDeleteStats;
 import org.apache.hadoop.hive.metastore.directsql.MetaStoreDirectSql;
+import org.apache.hadoop.hive.metastore.metastore.DirectSqlOnlyHelper;
 import org.apache.hadoop.hive.metastore.metastore.GetHelper;
 import org.apache.hadoop.hive.metastore.metastore.GetListHelper;
 import org.apache.hadoop.hive.metastore.metastore.RawStoreBundle;
@@ -125,20 +126,15 @@ public void setBaseStore(RawStore store) {
   
   @Override
   public List<TableName> getTableNamesWithStats() throws MetaException, 
NoSuchObjectException {
-    return new GetListHelper<TableName, TableName>(this, null) {
+    return new DirectSqlOnlyHelper<TableName, List<TableName>>(this, null) {
       @Override
       protected List<TableName> getSqlResult() throws MetaException {
         return getDirectSql().getTableNamesWithStats();
       }
 
       @Override
-      protected boolean canUseJdoQuery() throws MetaException {
-        return false;
-      }
-
-      @Override
-      protected List<TableName> getJdoResult() throws MetaException {
-        throw new 
UnsupportedOperationException("UnsupportedOperationException"); // TODO: 
implement?
+      protected String describeResult() {
+        return (results != null ? results.size() : 0) + " tables with column 
statistics";
       }
     }.run(false);
   }
@@ -149,7 +145,7 @@ public Map<String, List<String>> 
getPartitionColsWithStats(TableName tableName)
     String catName = normalizeIdentifier(tableName.getCat());
     String dbName = normalizeIdentifier(tableName.getDb());
     String tblName = normalizeIdentifier(tableName.getTable());
-    return new GetHelper<TableName, Map<String, List<String>>>(this, null) {
+    return new DirectSqlOnlyHelper<TableName, Map<String, List<String>>>(this, 
null) {
       @Override
       protected Map<String, List<String>> getSqlResult() throws MetaException {
         try {
@@ -160,16 +156,6 @@ protected Map<String, List<String>> getSqlResult() throws 
MetaException {
         }
       }
 
-      @Override
-      protected boolean canUseJdoQuery() throws MetaException {
-        return false;
-      }
-
-      @Override
-      protected Map<String, List<String>> getJdoResult() throws MetaException {
-        throw new 
UnsupportedOperationException("UnsupportedOperationException"); // TODO: 
implement?
-      }
-
       @Override
       protected String describeResult() {
         return results.size() + " partitions";
@@ -447,7 +433,7 @@ public Map<String, Map<String, String>> 
updatePartitionColumnStatisticsInBatch(
       String validWriteIds, long writeId)
       throws NoSuchObjectException, MetaException, InvalidObjectException, 
InvalidInputException {
 
-    return new GetHelper<TableName, Map<String, Map<String, String>>>(this, 
null) {
+    return new DirectSqlOnlyHelper<TableName, Map<String, Map<String, 
String>>>(this, null) {
       @Override
       protected String describeResult() {
         return "Map of partition key to column stats if successful";
@@ -458,17 +444,6 @@ protected Map<String, Map<String, String>> getSqlResult()
         return 
getDirectSql().updatePartitionColumnStatisticsBatch(partColStatsMap, tbl,
             listeners, validWriteIds, writeId);
       }
-
-      @Override
-      protected boolean canUseJdoQuery() throws MetaException {
-        return false;
-      }
-
-      @Override
-      protected Map<String, Map<String, String>> getJdoResult()
-          throws MetaException, NoSuchObjectException, InvalidObjectException, 
InvalidInputException {
-        throw new UnsupportedOperationException("Cannot update partition 
column statistics with JDO, make sure direct SQL is enabled");
-      }
     }.run(false);
   }
 
@@ -803,7 +778,7 @@ public AggrStats get_aggr_stats_for(TableName tableName,
     final double ndvTuner = MetastoreConf.getDoubleVar(conf, 
MetastoreConf.ConfVars.STATS_NDV_TUNER);
     final boolean enableBitVector = MetastoreConf.getBoolVar(conf, 
MetastoreConf.ConfVars.STATS_FETCH_BITVECTOR);
     final boolean enableKll = MetastoreConf.getBoolVar(conf, 
MetastoreConf.ConfVars.STATS_FETCH_KLL);
-    return new GetHelper<TableName, AggrStats>(this, tableName) {
+    return new DirectSqlOnlyHelper<TableName, AggrStats>(this, tableName) {
       @Override
       protected AggrStats getSqlResult()
           throws MetaException {
@@ -814,22 +789,9 @@ protected AggrStats getSqlResult()
             colNames, engine, useDensityFunctionForNDVEstimation, ndvTuner, 
enableBitVector, enableKll);
       }
 
-      @Override
-      protected boolean canUseJdoQuery() throws MetaException {
-        return false;
-      }
-
-      @Override
-      protected AggrStats getJdoResult()
-          throws MetaException, NoSuchObjectException {
-        // This is fast path for query optimizations, if we can find this info
-        // quickly using
-        // directSql, do it. No point in failing back to slow path here.
-        throw new MetaException("Jdo path is not implemented for stats aggr.");
-      }
       @Override
       protected String describeResult() {
-        return null;
+        return (results != null ? results.getColStatsSize() : 0) + " 
aggregated column statistics";
       }
     }.run(true);
   }
@@ -839,30 +801,16 @@ public 
List<MetaStoreServerUtils.ColStatsObjWithSourceInfo> getPartitionColStats
       throws MetaException, NoSuchObjectException {
     final boolean enableBitVector = MetastoreConf.getBoolVar(conf, 
MetastoreConf.ConfVars.STATS_FETCH_BITVECTOR);
     final boolean enableKll = MetastoreConf.getBoolVar(conf, 
MetastoreConf.ConfVars.STATS_FETCH_KLL);
-    return new GetHelper<TableName, 
List<MetaStoreServerUtils.ColStatsObjWithSourceInfo>>(this, null) {
+    return new DirectSqlOnlyHelper<TableName, 
List<MetaStoreServerUtils.ColStatsObjWithSourceInfo>>(this, null) {
       @Override
       protected List<MetaStoreServerUtils.ColStatsObjWithSourceInfo> 
getSqlResult() throws MetaException {
         return new DirectSqlAggrStats(pm, conf, schema)
             .getColStatsForAllTablePartitions(catName, dbName, 
enableBitVector, enableKll);
       }
 
-      @Override
-      protected boolean canUseJdoQuery() throws MetaException {
-        return false;
-      }
-
-      @Override
-      protected List<MetaStoreServerUtils.ColStatsObjWithSourceInfo> 
getJdoResult()
-          throws MetaException, NoSuchObjectException {
-        // This is fast path for query optimizations, if we can find this info
-        // quickly using directSql, do it. No point in failing back to slow 
path
-        // here.
-        throw new MetaException("Jdo path is not implemented for 
getPartitionColStatsForDatabase.");
-      }
-
       @Override
       protected String describeResult() {
-        return null;
+        return (results != null ? results.size() : 0) + " partition column 
statistics";
       }
     }.run(false);
   }
@@ -1186,17 +1134,15 @@ private void executePlainSQL(String sql,
   }
 
   private GetHelper<TableName, String> createSubTransactionHelper(TableName 
tableName) throws MetaException {
-    return new GetHelper<TableName, String>(this, tableName) {
+    return new GetHelper<>(this, tableName) {
       @Override
       protected String describeResult() {
         return "A helper for managing sub-transaction";
       }
-
       @Override
       protected String getSqlResult() {
         throw new UnsupportedOperationException();
       }
-
       @Override
       protected String getJdoResult() {
         throw new UnsupportedOperationException();
diff --git 
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/TableStoreImpl.java
 
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/TableStoreImpl.java
index 6caed023de5..bfcbd36da05 100644
--- 
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/TableStoreImpl.java
+++ 
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/TableStoreImpl.java
@@ -52,7 +52,6 @@
 import org.apache.hadoop.hive.metastore.Batchable;
 import org.apache.hadoop.hive.metastore.DatabaseProduct;
 import org.apache.hadoop.hive.metastore.Deadline;
-import org.apache.hadoop.hive.metastore.ObjectStore;
 import org.apache.hadoop.hive.metastore.api.Order;
 import org.apache.hadoop.hive.metastore.api.SerDeInfo;
 import org.apache.hadoop.hive.metastore.api.SkewedInfo;
@@ -115,6 +114,7 @@
 import org.apache.hadoop.hive.metastore.model.MTableColumnPrivilege;
 import org.apache.hadoop.hive.metastore.model.MTablePrivilege;
 import org.apache.hadoop.hive.metastore.parser.ExpressionTree;
+import org.apache.hadoop.hive.metastore.metastore.DirectSqlOnlyHelper;
 import org.apache.hadoop.hive.metastore.metastore.GetHelper;
 import org.apache.hadoop.hive.metastore.metastore.GetListHelper;
 import org.apache.hadoop.hive.metastore.metastore.iface.TableStore;
@@ -2859,7 +2859,7 @@ public int getObjectCount(String fieldName, String 
objName) {
   @Override
   public long updateParameterWithExpectedValue(Table table, String key, String 
expectedValue, String newValue)
       throws MetaException, NoSuchObjectException {
-    return new GetHelper<TableName, Long>(this,
+    return new DirectSqlOnlyHelper<TableName, Long>(this,
         new TableName(table.getCatName(), table.getDbName(), 
table.getTableName())) {
       @Override
       protected String describeResult() {
@@ -2869,16 +2869,6 @@ protected String describeResult() {
       protected Long getSqlResult() throws MetaException {
         return getDirectSql().updateTableParam(table, key, expectedValue, 
newValue);
       }
-      @Override
-      protected Long getJdoResult()
-          throws MetaException, NoSuchObjectException, InvalidObjectException {
-        throw new UnsupportedOperationException(
-            "Cannot update parameter with JDO, make sure direct SQL is 
enabled");
-      }
-      @Override
-      protected boolean canUseJdoQuery() throws MetaException {
-        return false;
-      }
     }.run(false);
   }
 
@@ -2907,19 +2897,16 @@ public void 
updateTableParams(List<Map.Entry<TableParamsUpdate, Table>> updates)
     if (updates == null || updates.isEmpty()) {
       return;
     }
-    new GetListHelper<TableName, Void>(this, null) {
+    new DirectSqlOnlyHelper<TableName, List<Void>>(this, null) {
       @Override
       protected List<Void> getSqlResult() throws MetaException {
         getDirectSql().updateTableParams(updates);
         return null;
       }
+
       @Override
-      protected boolean canUseJdoQuery() {
-        return false;
-      }
-      @Override
-      protected List<Void> getJdoResult() {
-        throw new 
UnsupportedOperationException("UnsupportedOperationException");
+      protected String describeResult() {
+        return updates.size() + " table(s) parameter updates";
       }
     }.run(false);
   }
diff --git 
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java
 
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java
index 1462b4ec59b..9beead8b6c1 100644
--- 
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java
+++ 
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java
@@ -66,6 +66,7 @@
 import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
 import org.apache.hadoop.hive.metastore.api.StoredProcedure;
 import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.api.TableParamsUpdate;
 import org.apache.hadoop.hive.metastore.client.builder.CatalogBuilder;
 import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder;
 import org.apache.hadoop.hive.metastore.client.builder.GetPartitionsArgs;
@@ -815,6 +816,53 @@ public void testDirectSQLDropPartitionsCacheInSession()
     Assert.assertEquals(1, partitions.size());
   }
 
+  /**
+   * Operations that only have a direct SQL implementation must keep working 
when
+   * metastore.try.direct.sql is false, as the setting only chooses between 
direct SQL and JDO
+   * where both exist.
+   */
+  @Test
+  public void testDirectSqlOnlyOperations() throws Exception {
+    createPartitionedTable(false, true, Collections.emptySet());
+    Table tbl;
+    try (AutoCloseable c = deadline()) {
+      tbl = objectStore.getTable(DEFAULT_CATALOG_NAME, DB1, TABLE1);
+    }
+    TableName tableName = new TableName(DEFAULT_CATALOG_NAME, DB1, TABLE1);
+    TableStore tableStore = objectStore.unwrap(TableStore.class);
+
+    try (AutoCloseable c = deadline();
+         AutoCloseable d = new DirectSqlConfigurator(conf, false)) {
+      Assert.assertFalse(MetastoreConf.getBoolVar(objectStore.getConf(), 
ConfVars.TRY_DIRECT_SQL));
+      // updateTableParams must persist the parameters
+      Map<String, String> params = new HashMap<>();
+      params.put("test_key", "v1");
+      tableStore.updateTableParams(Collections.singletonList(
+          Map.entry((new TableParamsUpdate(
+              new org.apache.hadoop.hive.metastore.api.TableName(DB1, TABLE1), 
params)), tbl)));
+      Assert.assertEquals("v1",
+          objectStore.getTable(DEFAULT_CATALOG_NAME, DB1, 
TABLE1).getParameters().get("test_key"));
+
+      // updateParameterWithExpectedValue: no NPE, and compare-and-set 
semantics are preserved
+      Assert.assertEquals(0, tableStore.updateParameterWithExpectedValue(tbl, 
"test_key", "wrong", "v2"));
+      Assert.assertEquals(1, tableStore.updateParameterWithExpectedValue(tbl, 
"test_key", "v1", "v2"));
+      Assert.assertEquals("v2",
+          objectStore.getTable(DEFAULT_CATALOG_NAME, DB1, 
TABLE1).getParameters().get("test_key"));
+
+      // the column statistics reads must not return null
+      Assert.assertNotNull(objectStore.getTableNamesWithStats());
+      
Assert.assertTrue(objectStore.getTableNamesWithStats().contains(tableName));
+      Map<String, List<String>> partCols = 
objectStore.getPartitionColsWithStats(DEFAULT_CATALOG_NAME, DB1, TABLE1);
+      Assert.assertNotNull(partCols);
+      Assert.assertEquals(3, partCols.size());
+      
Assert.assertNotNull(objectStore.getPartitionColStatsForDatabase(DEFAULT_CATALOG_NAME,
 DB1));
+      AggrStats aggrStats = 
objectStore.get_aggr_stats_for(DEFAULT_CATALOG_NAME, DB1, TABLE1, 
ALL_PART_NAMES,
+          Collections.singletonList("test_part_col"), ENGINE);
+      Assert.assertNotNull(aggrStats);
+      Assert.assertEquals(1, aggrStats.getColStatsSize());
+    }
+  }
+
   @Test
   public void testDirectSQLDropPartitionsRejectsSqlInjectionInPartName()
       throws Exception {

Reply via email to