vihangk1 commented on a change in pull request #1470:
URL: https://github.com/apache/hive/pull/1470#discussion_r499869127
##########
File path:
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java
##########
@@ -20,71 +20,666 @@
import java.sql.SQLException;
import java.sql.SQLTransactionRollbackException;
+import java.sql.Timestamp;
+import java.util.ArrayList;
+import java.util.EnumMap;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
-/** Database product infered via JDBC. */
-public enum DatabaseProduct {
- DERBY, MYSQL, POSTGRES, ORACLE, SQLSERVER, OTHER;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import com.google.common.base.Preconditions;
+
+/** Database product inferred via JDBC. Encapsulates all SQL logic associated
with
+ * the database product.
+ * This class is a singleton, which is instantiated the first time
+ * method determineDatabaseProduct is invoked.
+ * Tests that need to create multiple instances can use the reset method
+ * */
+public class DatabaseProduct implements Configurable {
+ static final private Logger LOG =
LoggerFactory.getLogger(DatabaseProduct.class.getName());
+
+ private static enum DbType {DERBY, MYSQL, POSTGRES, ORACLE, SQLSERVER,
CUSTOM, UNDEFINED};
+ public DbType dbType;
+
+ // Singleton instance
+ private static DatabaseProduct theDatabaseProduct;
+
+ Configuration myConf;
+ /**
+ * Protected constructor for singleton class
+ * @param id
+ */
+ protected DatabaseProduct() {}
+
+ public static final String DERBY_NAME = "derby";
+ public static final String SQL_SERVER_NAME = "microsoft sql server";
+ public static final String MYSQL_NAME = "mysql";
+ public static final String POSTGRESQL_NAME = "postgresql";
+ public static final String ORACLE_NAME = "oracle";
+ public static final String UNDEFINED_NAME = "other";
+
/**
* Determine the database product type
* @param productName string to defer database connection
* @return database product type
*/
- public static DatabaseProduct determineDatabaseProduct(String productName)
throws SQLException {
- if (productName == null) {
- return OTHER;
+ public static DatabaseProduct determineDatabaseProduct(String productName,
Configuration c) {
+ DbType dbt;
+
+ if (theDatabaseProduct != null) {
+ Preconditions.checkState(theDatabaseProduct.dbType ==
getDbType(productName));
+ return theDatabaseProduct;
}
+
+ // This method may be invoked by concurrent connections
+ synchronized (DatabaseProduct.class) {
+
+ if (productName == null) {
+ productName = UNDEFINED_NAME;
+ }
+
+ dbt = getDbType(productName);
+
+ // Check for null again in case of race condition
+ if (theDatabaseProduct == null) {
+ final Configuration conf = c!= null ? c :
MetastoreConf.newMetastoreConf();
+ // Check if we are using an external database product
+ boolean isExternal = MetastoreConf.getBoolVar(conf,
ConfVars.USE_CUSTOM_RDBMS);
+
+ if (isExternal) {
+ // The DatabaseProduct will be created by instantiating an external
class via
+ // reflection. The external class can override any method in the
current class
+ String className = MetastoreConf.getVar(conf,
ConfVars.CUSTOM_RDBMS_CLASSNAME);
+
+ if (className != null) {
+ try {
+ theDatabaseProduct = (DatabaseProduct)
+ ReflectionUtils.newInstance(Class.forName(className), conf);
+
+ LOG.info(String.format("Using custom RDBMS %s. Overriding
DbType: %s", className, dbt));
Review comment:
The Overriding DbType: is bit confusing. Why is that log useful?
##########
File path:
standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
##########
@@ -1337,6 +1337,15 @@ public static ConfVars getMetaConf(String name) {
HIVE_TXN_STATS_ENABLED("hive.txn.stats.enabled", "hive.txn.stats.enabled",
true,
"Whether Hive supports transactional stats (accurate stats for
transactional tables)"),
+ // External RDBMS support
+ USE_CUSTOM_RDBMS("metastore.use.custom.database.product",
+ "hive.metastore.use.custom.database.product", false,
+ "Use an external RDBMS for the metastore"),
Review comment:
Its interesting that I don't see "hive.metastore.db.type" configuration
in MetastoreConf which I think is an unrelated bug. It would be great if you
could add more details here. May be say "Use a external RDBMS which not in the
list of natively supported databases eg. Derby, Mysql, Oracle, Postgres, MSSQL
as defined by hive.metastore.db.type. If this configuration is true the
metastore.custom.database.product.classname must be set to a valid classname."
##########
File path:
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java
##########
@@ -20,71 +20,666 @@
import java.sql.SQLException;
import java.sql.SQLTransactionRollbackException;
+import java.sql.Timestamp;
+import java.util.ArrayList;
+import java.util.EnumMap;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
-/** Database product infered via JDBC. */
-public enum DatabaseProduct {
- DERBY, MYSQL, POSTGRES, ORACLE, SQLSERVER, OTHER;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import com.google.common.base.Preconditions;
+
+/** Database product inferred via JDBC. Encapsulates all SQL logic associated
with
+ * the database product.
+ * This class is a singleton, which is instantiated the first time
+ * method determineDatabaseProduct is invoked.
+ * Tests that need to create multiple instances can use the reset method
+ * */
+public class DatabaseProduct implements Configurable {
+ static final private Logger LOG =
LoggerFactory.getLogger(DatabaseProduct.class.getName());
+
+ private static enum DbType {DERBY, MYSQL, POSTGRES, ORACLE, SQLSERVER,
CUSTOM, UNDEFINED};
+ public DbType dbType;
+
+ // Singleton instance
+ private static DatabaseProduct theDatabaseProduct;
+
+ Configuration myConf;
+ /**
+ * Protected constructor for singleton class
+ * @param id
+ */
+ protected DatabaseProduct() {}
+
+ public static final String DERBY_NAME = "derby";
+ public static final String SQL_SERVER_NAME = "microsoft sql server";
+ public static final String MYSQL_NAME = "mysql";
+ public static final String POSTGRESQL_NAME = "postgresql";
+ public static final String ORACLE_NAME = "oracle";
+ public static final String UNDEFINED_NAME = "other";
+
/**
* Determine the database product type
* @param productName string to defer database connection
* @return database product type
*/
- public static DatabaseProduct determineDatabaseProduct(String productName)
throws SQLException {
- if (productName == null) {
- return OTHER;
+ public static DatabaseProduct determineDatabaseProduct(String productName,
Configuration c) {
+ DbType dbt;
+
+ if (theDatabaseProduct != null) {
+ Preconditions.checkState(theDatabaseProduct.dbType ==
getDbType(productName));
+ return theDatabaseProduct;
}
+
+ // This method may be invoked by concurrent connections
+ synchronized (DatabaseProduct.class) {
+
+ if (productName == null) {
+ productName = UNDEFINED_NAME;
+ }
+
+ dbt = getDbType(productName);
+
+ // Check for null again in case of race condition
+ if (theDatabaseProduct == null) {
+ final Configuration conf = c!= null ? c :
MetastoreConf.newMetastoreConf();
+ // Check if we are using an external database product
+ boolean isExternal = MetastoreConf.getBoolVar(conf,
ConfVars.USE_CUSTOM_RDBMS);
+
+ if (isExternal) {
+ // The DatabaseProduct will be created by instantiating an external
class via
+ // reflection. The external class can override any method in the
current class
+ String className = MetastoreConf.getVar(conf,
ConfVars.CUSTOM_RDBMS_CLASSNAME);
+
+ if (className != null) {
+ try {
+ theDatabaseProduct = (DatabaseProduct)
+ ReflectionUtils.newInstance(Class.forName(className), conf);
+
+ LOG.info(String.format("Using custom RDBMS %s. Overriding
DbType: %s", className, dbt));
+ dbt = DbType.CUSTOM;
+ }catch (Exception e) {
+ LOG.warn("Caught exception instantiating custom database
product. Reverting to " + dbt, e);
+ }
+ }
+ else {
+ LOG.warn("Unexpected: metastore.use.custom.database.product was
set, " +
+ "but metastore.custom.database.product.classname was not.
Reverting to " + dbt);
+ }
+ }
+
+ if (theDatabaseProduct == null) {
+ theDatabaseProduct = new DatabaseProduct();
Review comment:
It looks like when this line is executed the dbType is null. Do you need
to initialize a default value of dbType = UNDEFINED?
##########
File path:
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java
##########
@@ -20,71 +20,646 @@
import java.sql.SQLException;
import java.sql.SQLTransactionRollbackException;
+import java.sql.Timestamp;
+import java.util.ArrayList;
+import java.util.EnumMap;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
-/** Database product infered via JDBC. */
-public enum DatabaseProduct {
- DERBY, MYSQL, POSTGRES, ORACLE, SQLSERVER, OTHER;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+/** Database product inferred via JDBC. Encapsulates all SQL logic associated
with
+ * the database product.
+ * This class is a singleton, which is instantiated the first time
+ * method determineDatabaseProduct is invoked.
+ * Tests that need to create multiple instances can use the reset method
+ * */
+public class DatabaseProduct implements Configurable {
+ static final private Logger LOG =
LoggerFactory.getLogger(DatabaseProduct.class.getName());
+
+ private static enum DbType {DERBY, MYSQL, POSTGRES, ORACLE, SQLSERVER,
EXTERNAL, OTHER};
+ public DbType dbType;
+
+ // Singleton instance
+ private static DatabaseProduct theDatabaseProduct;
+
+ static {
+ final Configuration conf = MetastoreConf.newMetastoreConf();
+ // Check if we are using an external database product
+ boolean isExternal = MetastoreConf.getBoolVar(conf,
ConfVars.USE_CUSTOM_RDBMS);
+
+ if (isExternal) {
+ // The DatabaseProduct will be created by instantiating an external
class via
+ // reflection. The external class can override any method in the current
class
+ String className = MetastoreConf.getVar(conf,
ConfVars.CUSTOM_RDBMS_CLASSNAME);
+
+ if (className != null) {
+ try {
+ theDatabaseProduct = (DatabaseProduct)
+ ReflectionUtils.newInstance(Class.forName(className), conf);
+
+ theDatabaseProduct.dbType = DbType.EXTERNAL;
+ }catch (Exception e) {
+ LOG.warn("Unable to instantiate custom database product. Reverting
to default", e);
+ }
+ }
+ else {
+ LOG.warn("metastore.use.custom.database.product was set, " +
+ "but metastore.custom.database.product.classname was not.
Reverting to default");
+ }
+ }
+ }
+
+ /**
+ * Private constructor for singleton class
+ * @param id
+ */
+ private DatabaseProduct() {}
+
+ public static final String DERBY_NAME = "derby";
+ public static final String SQL_SERVER_NAME = "microsoft sql server";
+ public static final String MYSQL_NAME = "mysql";
+ public static final String POSTGRESQL_NAME = "postgresql";
+ public static final String ORACLE_NAME = "oracle";
+ public static final String OTHER_NAME = "other";
+
/**
* Determine the database product type
* @param productName string to defer database connection
* @return database product type
*/
- public static DatabaseProduct determineDatabaseProduct(String productName)
throws SQLException {
+ public static DatabaseProduct determineDatabaseProduct(String productName) {
+ DbType dbt;
+
if (productName == null) {
- return OTHER;
+ productName = OTHER_NAME;
}
+
productName = productName.toLowerCase();
- if (productName.contains("derby")) {
- return DERBY;
- } else if (productName.contains("microsoft sql server")) {
- return SQLSERVER;
- } else if (productName.contains("mysql")) {
- return MYSQL;
- } else if (productName.contains("oracle")) {
- return ORACLE;
- } else if (productName.contains("postgresql")) {
- return POSTGRES;
+
+ if (productName.contains(DERBY_NAME)) {
+ dbt = DbType.DERBY;
+ } else if (productName.contains(SQL_SERVER_NAME)) {
+ dbt = DbType.SQLSERVER;
+ } else if (productName.contains(MYSQL_NAME)) {
+ dbt = DbType.MYSQL;
+ } else if (productName.contains(ORACLE_NAME)) {
+ dbt = DbType.ORACLE;
+ } else if (productName.contains(POSTGRESQL_NAME)) {
+ dbt = DbType.POSTGRES;
} else {
- return OTHER;
+ dbt = DbType.OTHER;
}
+
+ // This method may be invoked by concurrent connections
+ synchronized (DatabaseProduct.class) {
+ if (theDatabaseProduct == null) {
+ theDatabaseProduct = new DatabaseProduct();
+ }
+
+ theDatabaseProduct.dbType = dbt;
+ }
+ return theDatabaseProduct;
+ }
+
+ public final boolean isDERBY() {
+ return dbType == DbType.DERBY;
+ }
+
+ public final boolean isMYSQL() {
+ return dbType == DbType.MYSQL;
}
- public static boolean isDeadlock(DatabaseProduct dbProduct, SQLException e) {
+ public final boolean isORACLE() {
+ return dbType == DbType.ORACLE;
+ }
+
+ public final boolean isSQLSERVER() {
+ return dbType == DbType.SQLSERVER;
+ }
+
+ public final boolean isPOSTGRES() {
+ return dbType == DbType.POSTGRES;
+ }
+
+ public final boolean isEXTERNAL() {
+ return dbType == DbType.EXTERNAL;
+ }
+
+ public final boolean isOTHER() {
+ return dbType == DbType.OTHER;
+ }
+
+ public boolean isDeadlock(SQLException e) {
return e instanceof SQLTransactionRollbackException
- || ((dbProduct == MYSQL || dbProduct == POSTGRES || dbProduct ==
SQLSERVER)
+ || ((isMYSQL() || isPOSTGRES() || isSQLSERVER() || isEXTERNAL())
&& "40001".equals(e.getSQLState()))
- || (dbProduct == POSTGRES && "40P01".equals(e.getSQLState()))
- || (dbProduct == ORACLE && (e.getMessage() != null &&
(e.getMessage().contains("deadlock detected")
+ || (isPOSTGRES() && "40P01".equals(e.getSQLState()))
+ || (isORACLE() && (e.getMessage() != null &&
(e.getMessage().contains("deadlock detected")
|| e.getMessage().contains("can't serialize access for this
transaction"))));
}
/**
* Whether the RDBMS has restrictions on IN list size (explicit, or poor
perf-based).
*/
- public static boolean needsInBatching(DatabaseProduct dbType) {
- return dbType == ORACLE || dbType == SQLSERVER;
+ protected boolean needsInBatching() {
+ return isORACLE() || isSQLSERVER();
}
/**
* Whether the RDBMS has a bug in join and filter operation order described
in DERBY-6358.
*/
- public static boolean hasJoinOperationOrderBug(DatabaseProduct dbType) {
- return dbType == DERBY || dbType == ORACLE || dbType == POSTGRES;
+ protected boolean hasJoinOperationOrderBug() {
+ return isDERBY() || isORACLE() || isPOSTGRES();
}
- public static String getHiveSchemaPostfix(DatabaseProduct dbType) {
+ public String getHiveSchemaPostfix() {
switch (dbType) {
case SQLSERVER:
return "mssql";
case DERBY:
case MYSQL:
case POSTGRES:
case ORACLE:
+ case EXTERNAL:
return dbType.name().toLowerCase();
case OTHER:
default:
return null;
}
}
+
+ public static void reset() {
Review comment:
Thanks for the context. In such a case I think it would be good to
annotate this method as @VisibleForTesting
##########
File path:
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java
##########
@@ -20,71 +20,666 @@
import java.sql.SQLException;
import java.sql.SQLTransactionRollbackException;
+import java.sql.Timestamp;
+import java.util.ArrayList;
+import java.util.EnumMap;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
-/** Database product infered via JDBC. */
-public enum DatabaseProduct {
- DERBY, MYSQL, POSTGRES, ORACLE, SQLSERVER, OTHER;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import com.google.common.base.Preconditions;
+
+/** Database product inferred via JDBC. Encapsulates all SQL logic associated
with
+ * the database product.
+ * This class is a singleton, which is instantiated the first time
+ * method determineDatabaseProduct is invoked.
+ * Tests that need to create multiple instances can use the reset method
+ * */
+public class DatabaseProduct implements Configurable {
+ static final private Logger LOG =
LoggerFactory.getLogger(DatabaseProduct.class.getName());
+
+ private static enum DbType {DERBY, MYSQL, POSTGRES, ORACLE, SQLSERVER,
CUSTOM, UNDEFINED};
+ public DbType dbType;
+
+ // Singleton instance
+ private static DatabaseProduct theDatabaseProduct;
+
+ Configuration myConf;
+ /**
+ * Protected constructor for singleton class
+ * @param id
+ */
+ protected DatabaseProduct() {}
+
+ public static final String DERBY_NAME = "derby";
+ public static final String SQL_SERVER_NAME = "microsoft sql server";
+ public static final String MYSQL_NAME = "mysql";
+ public static final String POSTGRESQL_NAME = "postgresql";
+ public static final String ORACLE_NAME = "oracle";
+ public static final String UNDEFINED_NAME = "other";
+
/**
* Determine the database product type
* @param productName string to defer database connection
* @return database product type
*/
- public static DatabaseProduct determineDatabaseProduct(String productName)
throws SQLException {
- if (productName == null) {
- return OTHER;
+ public static DatabaseProduct determineDatabaseProduct(String productName,
Configuration c) {
+ DbType dbt;
+
+ if (theDatabaseProduct != null) {
+ Preconditions.checkState(theDatabaseProduct.dbType ==
getDbType(productName));
+ return theDatabaseProduct;
}
+
+ // This method may be invoked by concurrent connections
+ synchronized (DatabaseProduct.class) {
+
+ if (productName == null) {
+ productName = UNDEFINED_NAME;
+ }
+
+ dbt = getDbType(productName);
+
+ // Check for null again in case of race condition
+ if (theDatabaseProduct == null) {
+ final Configuration conf = c!= null ? c :
MetastoreConf.newMetastoreConf();
+ // Check if we are using an external database product
+ boolean isExternal = MetastoreConf.getBoolVar(conf,
ConfVars.USE_CUSTOM_RDBMS);
+
+ if (isExternal) {
+ // The DatabaseProduct will be created by instantiating an external
class via
+ // reflection. The external class can override any method in the
current class
+ String className = MetastoreConf.getVar(conf,
ConfVars.CUSTOM_RDBMS_CLASSNAME);
+
+ if (className != null) {
+ try {
+ theDatabaseProduct = (DatabaseProduct)
+ ReflectionUtils.newInstance(Class.forName(className), conf);
+
+ LOG.info(String.format("Using custom RDBMS %s. Overriding
DbType: %s", className, dbt));
+ dbt = DbType.CUSTOM;
+ }catch (Exception e) {
+ LOG.warn("Caught exception instantiating custom database
product. Reverting to " + dbt, e);
+ }
+ }
+ else {
+ LOG.warn("Unexpected: metastore.use.custom.database.product was
set, " +
Review comment:
I believe its better to throw an exception here with the appropriate
message.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
