This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 102b89e6816 HIVE-27762: Don't fall back to jdo query in ObjectStore if
direct sql throws unrecoverable exception (Wechar Yu, reviewed by Sai Hemanth
Gantasala, Denys Kuzmenko)
102b89e6816 is described below
commit 102b89e6816a7756cb50f2cc3889351dc0e270b2
Author: Wechar Yu <[email protected]>
AuthorDate: Sat Nov 11 03:01:32 2023 +0800
HIVE-27762: Don't fall back to jdo query in ObjectStore if direct sql
throws unrecoverable exception (Wechar Yu, reviewed by Sai Hemanth Gantasala,
Denys Kuzmenko)
Closes #4767
---
.../hadoop/hive/metastore/DatabaseProduct.java | 12 +++++++
.../apache/hadoop/hive/metastore/ObjectStore.java | 9 ++---
.../hadoop/hive/metastore/RetryingHMSHandler.java | 11 +-----
.../hadoop/hive/metastore/TestObjectStore.java | 40 ++++++++++++++++++++++
4 files changed, 56 insertions(+), 16 deletions(-)
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java
index 3f3d361b9a0..afee0797420 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hive.metastore;
import java.sql.SQLException;
+import java.sql.SQLIntegrityConstraintViolationException;
import java.sql.SQLTransactionRollbackException;
import java.sql.Timestamp;
import java.util.ArrayList;
@@ -26,7 +27,9 @@ import java.util.EnumMap;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.stream.Stream;
+import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.metastore.api.MetaException;
@@ -47,6 +50,10 @@ import com.google.common.base.Preconditions;
* */
public class DatabaseProduct implements Configurable {
static final private Logger LOG =
LoggerFactory.getLogger(DatabaseProduct.class.getName());
+ private static final Class<SQLException>[] unrecoverableSqlExceptions = new
Class[]{
+ // TODO: collect more unrecoverable SQLExceptions
+ SQLIntegrityConstraintViolationException.class
+ };
public enum DbType {DERBY, MYSQL, POSTGRES, ORACLE, SQLSERVER, CUSTOM,
UNDEFINED};
public DbType dbType;
@@ -154,6 +161,11 @@ public class DatabaseProduct implements Configurable {
return dbt;
}
+ public static boolean isRecoverableException(Throwable t) {
+ return Stream.of(unrecoverableSqlExceptions)
+ .allMatch(ex -> ExceptionUtils.indexOfType(t, ex) < 0);
+ }
+
public final boolean isDERBY() {
return dbType == DbType.DERBY;
}
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index fa132629934..98f7f6f85be 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -4431,6 +4431,9 @@ public class ObjectStore implements RawStore,
Configurable {
}
private void handleDirectSqlError(Exception ex, String savePoint) throws
MetaException, NoSuchObjectException {
+ if (!allowJdo || !DatabaseProduct.isRecoverableException(ex)) {
+ throw ExceptionHandler.newMetaException(ex);
+ }
String message = null;
try {
message = generateShorterMessage(ex);
@@ -4439,12 +4442,6 @@ public class ObjectStore implements RawStore,
Configurable {
}
LOG.warn(message); // Don't log the exception, people just get confused.
LOG.debug("Full DirectSQL callstack for debugging (not an error)", ex);
- if (!allowJdo) {
- if (ex instanceof MetaException) {
- throw (MetaException)ex;
- }
- throw new MetaException(ex.getMessage());
- }
if (!isInTxn) {
JDOException rollbackEx = null;
try {
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java
index ce7656fdb0c..0d52107ad09 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java
@@ -21,10 +21,7 @@ package org.apache.hadoop.hive.metastore;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.lang.reflect.UndeclaredThrowableException;
-import java.sql.SQLException;
-import java.sql.SQLIntegrityConstraintViolationException;
import java.util.concurrent.TimeUnit;
-import java.util.stream.Stream;
import javax.jdo.JDOException;
@@ -44,10 +41,6 @@ import org.datanucleus.exceptions.NucleusException;
@InterfaceStability.Evolving
public class RetryingHMSHandler extends AbstractHMSHandlerProxy {
private static final Logger LOG =
LoggerFactory.getLogger(RetryingHMSHandler.class);
- private static final Class<SQLException>[] unrecoverableSqlExceptions = new
Class[]{
- // TODO: collect more unrecoverable SQLExceptions
- SQLIntegrityConstraintViolationException.class
- };
private final long retryInterval;
private final int retryLimit;
@@ -185,8 +178,6 @@ public class RetryingHMSHandler extends
AbstractHMSHandlerProxy {
if (!(t instanceof JDOException || t instanceof NucleusException)) {
return false;
}
-
- return Stream.of(unrecoverableSqlExceptions)
- .allMatch(ex -> ExceptionUtils.indexOfType(t, ex) < 0);
+ return DatabaseProduct.isRecoverableException(t);
}
}
diff --git
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java
index 5746a952649..940b18d1db4 100644
---
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java
+++
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java
@@ -97,6 +97,7 @@ import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
+import java.sql.SQLIntegrityConstraintViolationException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.Arrays;
@@ -116,6 +117,7 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import static java.util.concurrent.Executors.newFixedThreadPool;
@@ -1631,6 +1633,44 @@ public class TestObjectStore {
Assert.assertEquals(0, objectStore.getPartitionCount());
}
+ @Test
+ public void testNoJdoForUnrecoverableException() throws Exception {
+ objectStore.openTransaction();
+ AtomicBoolean runDirectSql = new AtomicBoolean(false);
+ AtomicBoolean runJdo = new AtomicBoolean(false);
+ try {
+ objectStore.new GetHelper<Object>(DEFAULT_CATALOG_NAME, DB1, TABLE1,
true, true) {
+ @Override
+ protected String describeResult() {
+ return "test not run jdo for unrecoverable exception";
+ }
+
+ @Override
+ protected Object getSqlResult(ObjectStore.GetHelper ctx) throws
MetaException {
+ runDirectSql.set(true);
+ SQLIntegrityConstraintViolationException ex = new
SQLIntegrityConstraintViolationException("Unrecoverable ex");
+ MetaException me = new MetaException("Throwing unrecoverable
exception to test not run jdo.");
+ me.initCause(ex);
+ throw me;
+ }
+
+ @Override
+ protected Object getJdoResult(ObjectStore.GetHelper ctx) throws
MetaException, NoSuchObjectException {
+ runJdo.set(true);
+ SQLIntegrityConstraintViolationException ex = new
SQLIntegrityConstraintViolationException("Unrecoverable ex");
+ MetaException me = new MetaException("Throwing unrecoverable
exception to test not run jdo.");
+ me.initCause(ex);
+ throw me;
+ }
+ }.run(false);
+ } catch (MetaException ex) {
+ // expected
+ }
+ objectStore.commitTransaction();
+ Assert.assertEquals(true, runDirectSql.get());
+ Assert.assertEquals(false, runJdo.get());
+ }
+
/**
* Helper method to check whether the Java system properties were set
correctly in {@link ObjectStore#configureSSL(Configuration)}
* @param useSSL whether or not SSL is enabled
