This is an automated email from the ASF dual-hosted git repository.
ngangam pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 57322682d88 HIVE-26045: Detect timed out connections for providers and
auto-reconnect (#3595)
57322682d88 is described below
commit 57322682d88f4460da65fd88fced9754a53cb19c
Author: Butao Zhang <[email protected]>
AuthorDate: Thu Oct 20 13:28:01 2022 +0800
HIVE-26045: Detect timed out connections for providers and auto-reconnect
(#3595)
* HIVE-26045: Detect timed out connections for providers and auto-reconnect
* Address review comments
---
.../queries/clientpositive/dataconnector_mysql.q | 5 ++-
.../clientpositive/llap/dataconnector_mysql.q.out | 10 ++++-
.../AbstractDataConnectorProvider.java | 1 -
.../jdbc/AbstractJDBCConnectorProvider.java | 47 ++++++++++++++++++++--
4 files changed, 55 insertions(+), 8 deletions(-)
diff --git a/ql/src/test/queries/clientpositive/dataconnector_mysql.q
b/ql/src/test/queries/clientpositive/dataconnector_mysql.q
index 757c2f77387..78a43ee504a 100644
--- a/ql/src/test/queries/clientpositive/dataconnector_mysql.q
+++ b/ql/src/test/queries/clientpositive/dataconnector_mysql.q
@@ -6,7 +6,10 @@ URL 'jdbc:mysql://localhost:3306/qtestDB'
COMMENT 'test connector'
WITH DCPROPERTIES (
"hive.sql.dbcp.username"="root",
-"hive.sql.dbcp.password"="qtestpassword");
+"hive.sql.dbcp.password"="qtestpassword",
+"hive.connector.autoReconnect"="true",
+"hive.connector.maxReconnects"="3",
+"hive.connector.connectTimeout"="10000");
SHOW CONNECTORS;
CREATE REMOTE DATABASE db_mysql USING mysql_qtest with
DBPROPERTIES("connector.remoteDbName"="qtestDB");
diff --git a/ql/src/test/results/clientpositive/llap/dataconnector_mysql.q.out
b/ql/src/test/results/clientpositive/llap/dataconnector_mysql.q.out
index 351f6d2fea7..7912b07f7e7 100644
--- a/ql/src/test/results/clientpositive/llap/dataconnector_mysql.q.out
+++ b/ql/src/test/results/clientpositive/llap/dataconnector_mysql.q.out
@@ -4,7 +4,10 @@ URL 'jdbc:mysql://localhost:3306/qtestDB'
COMMENT 'test connector'
WITH DCPROPERTIES (
"hive.sql.dbcp.username"="root",
-"hive.sql.dbcp.password"="qtestpassword")
+"hive.sql.dbcp.password"="qtestpassword",
+"hive.connector.autoReconnect"="true",
+"hive.connector.maxReconnects"="3",
+"hive.connector.connectTimeout"="10000")
PREHOOK: type: CREATEDATACONNECTOR
PREHOOK: Output: connector:mysql_qtest
POSTHOOK: query: CREATE CONNECTOR mysql_qtest
@@ -13,7 +16,10 @@ URL 'jdbc:mysql://localhost:3306/qtestDB'
COMMENT 'test connector'
WITH DCPROPERTIES (
"hive.sql.dbcp.username"="root",
-"hive.sql.dbcp.password"="qtestpassword")
+"hive.sql.dbcp.password"="qtestpassword",
+"hive.connector.autoReconnect"="true",
+"hive.connector.maxReconnects"="3",
+"hive.connector.connectTimeout"="10000")
POSTHOOK: type: CREATEDATACONNECTOR
POSTHOOK: Output: connector:mysql_qtest
PREHOOK: query: SHOW CONNECTORS
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/dataconnector/AbstractDataConnectorProvider.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/dataconnector/AbstractDataConnectorProvider.java
index 295643aa2e4..cbb832f128c 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/dataconnector/AbstractDataConnectorProvider.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/dataconnector/AbstractDataConnectorProvider.java
@@ -36,7 +36,6 @@ import java.util.Map;
public abstract class AbstractDataConnectorProvider implements
IDataConnectorProvider {
protected String scoped_db = null;
protected Object handle = null;
- protected boolean isOpen = false;
protected DataConnector connector = null;
protected String driverClassName = null;
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/dataconnector/jdbc/AbstractJDBCConnectorProvider.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/dataconnector/jdbc/AbstractJDBCConnectorProvider.java
index d392a2567ae..aa322266fc3 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/dataconnector/jdbc/AbstractJDBCConnectorProvider.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/dataconnector/jdbc/AbstractJDBCConnectorProvider.java
@@ -37,8 +37,10 @@ import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Properties;
public abstract class AbstractJDBCConnectorProvider extends
AbstractDataConnectorProvider {
private static Logger LOG =
LoggerFactory.getLogger(AbstractJDBCConnectorProvider.class);
@@ -66,6 +68,9 @@ public abstract class AbstractJDBCConnectorProvider extends
AbstractDataConnecto
public static final String JDBC_NUM_PARTITIONS = JDBC_CONFIG_PREFIX +
".numPartitions";
public static final String JDBC_LOW_BOUND = JDBC_CONFIG_PREFIX +
".lowerBound";
public static final String JDBC_UPPER_BOUND = JDBC_CONFIG_PREFIX +
".upperBound";
+ // JDBC_CONNECTOR_PREFIX is used for indicating connection properties for
different connectors,
+ // such as hive.connector.autoReconnect in which autoReconnect is the jdbc
connection property.
+ public static final String JDBC_CONNECTOR_PREFIX = "hive.connector.";
private static final String JDBC_INPUTFORMAT_CLASS =
"org.apache.hive.storage.jdbc.JdbcInputFormat".intern();
private static final String JDBC_OUTPUTFORMAT_CLASS =
"org.apache.hive.storage.jdbc.JdbcOutputFormat".intern();
@@ -74,6 +79,7 @@ public abstract class AbstractJDBCConnectorProvider extends
AbstractDataConnecto
String jdbcUrl = null;
String username = null;
String password = null; // TODO convert to byte array
+ Map<String, String> connectorPropMap = new HashMap<>();
public AbstractJDBCConnectorProvider(String dbName, DataConnector dataConn,
String driverClass) {
super(dbName, dataConn, driverClass);
@@ -81,6 +87,11 @@ public abstract class AbstractJDBCConnectorProvider extends
AbstractDataConnecto
this.jdbcUrl = connector.getUrl();
this.username = connector.getParameters().get(JDBC_USERNAME);
this.password = connector.getParameters().get(JDBC_PASSWORD);
+ connector.getParameters().forEach((k, v) -> {
+ if (k.startsWith(JDBC_CONNECTOR_PREFIX))
+ connectorPropMap.put(k.substring(15), v);
+ });
+
if (this.password == null) {
String keystore = connector.getParameters().get(JDBC_KEYSTORE);
String key = connector.getParameters().get(JDBC_KEY);
@@ -107,8 +118,8 @@ public abstract class AbstractJDBCConnectorProvider extends
AbstractDataConnecto
@Override public void open() throws ConnectException {
try {
- handle = DriverManager.getConnection(jdbcUrl, username, password);
- isOpen = true;
+ close();
+ handle = DriverManager.getDriver(jdbcUrl).connect(jdbcUrl,
getConnectionProperties());
} catch (SQLException sqle) {
LOG.warn("Could not connect to remote data source at " + jdbcUrl);
throw new ConnectException("Could not connect to remote datasource at "
+ jdbcUrl + ",cause:" + sqle.getMessage());
@@ -117,7 +128,7 @@ public abstract class AbstractJDBCConnectorProvider extends
AbstractDataConnecto
protected Connection getConnection() {
try {
- if (!isOpen)
+ if (!isOpen())
open();
} catch (ConnectException ce) {
throw new RuntimeException(ce.getMessage(), ce);
@@ -129,8 +140,28 @@ public abstract class AbstractJDBCConnectorProvider
extends AbstractDataConnecto
throw new RuntimeException("unexpected type for connection handle");
}
+ protected boolean isOpen() {
+ try {
+ if (handle instanceof Connection)
+ return ((Connection) handle).isValid(3);
+ } catch (SQLException e) {
+ LOG.warn("Could not validate jdbc connection to "+ jdbcUrl, e);
+ }
+ return false;
+ }
+
+ protected boolean isClosed() {
+ try {
+ if (handle instanceof Connection)
+ return ((Connection) handle).isClosed();
+ } catch (SQLException e) {
+ LOG.warn("Could not determine whether jdbc connection is closed or not
to "+ jdbcUrl, e);
+ }
+ return true;
+ }
+
@Override public void close() {
- if (isOpen) {
+ if (!isClosed()) {
try {
((Connection)handle).close();
} catch (SQLException sqle) {
@@ -373,4 +404,12 @@ public abstract class AbstractJDBCConnectorProvider
extends AbstractDataConnecto
}
return "some_dummy_path";
}
+
+ protected Properties getConnectionProperties() {
+ Properties connectionProperties = new Properties();
+ connectionProperties.setProperty("user", username);
+ connectionProperties.setProperty("password", password);
+ connectorPropMap.forEach((k, v) -> connectionProperties.put(k, v));
+ return connectionProperties;
+ }
}
\ No newline at end of file
