Author: khorgath Date: Thu Oct 16 17:17:11 2014 New Revision: 1632389 URL: http://svn.apache.org/r1632389 Log: HIVE-7366 : getDatabase using direct sql (Sushanth Sowmyan, reviewed by Sergei Shelukhin)
Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java?rev=1632389&r1=1632388&r2=1632389&view=diff ============================================================================== --- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java (original) +++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java Thu Oct 16 17:17:11 2014 @@ -45,10 +45,13 @@ import org.apache.hadoop.hive.metastore. import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; +import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.Order; import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.PrincipalType; import org.apache.hadoop.hive.metastore.api.SerDeInfo; import org.apache.hadoop.hive.metastore.api.SkewedInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; @@ -143,13 +146,18 @@ class MetaStoreDirectSql { } /** - * See {@link #trySetAnsiQuotesForMysql()}. + * This function is intended to be called by functions before they put together a query + * Thus, any query-specific instantiation to be done from within the transaction is done + * here - for eg., for MySQL, we signal that we want to use ANSI SQL quoting behaviour */ - private void setAnsiQuotesForMysql() throws MetaException { - try { - trySetAnsiQuotesForMysql(); - } catch (SQLException sqlEx) { - throw new MetaException("Error setting ansi quotes: " + sqlEx.getMessage()); + private void doDbSpecificInitializationsBeforeQuery() throws MetaException { + if (isMySql){ + try { + assert pm.currentTransaction().isActive(); // must be inside tx together with queries + trySetAnsiQuotesForMysql(); + } catch (SQLException sqlEx) { + throw new MetaException("Error setting ansi quotes: " + sqlEx.getMessage()); + } } } @@ -171,6 +179,78 @@ class MetaStoreDirectSql { } } + public Database getDatabase(String dbName) throws MetaException{ + Query queryDbSelector = null; + Query queryDbParams = null; + try { + dbName = dbName.toLowerCase(); + + doDbSpecificInitializationsBeforeQuery(); + + String queryTextDbSelector= "select " + + "\"DB_ID\", \"NAME\", \"DB_LOCATION_URI\", \"DESC\", " + + "\"OWNER_NAME\", \"OWNER_TYPE\" " + + "FROM \"DBS\" where \"NAME\" = ? "; + Object[] params = new Object[] { dbName }; + queryDbSelector = pm.newQuery("javax.jdo.query.SQL", queryTextDbSelector); + + LOG.debug("getDatabase:query instantiated : " + queryTextDbSelector + " with param ["+params[0]+"]"); + + List<Object[]> sqlResult = (List<Object[]>)queryDbSelector.executeWithArray(params); + if ((sqlResult == null) || sqlResult.isEmpty()) { + LOG.debug("getDatabase:queryDbSelector ran, returned no/empty results, returning NoSuchObjectException"); + throw new MetaException("There is no database named " + dbName); + } + + assert(sqlResult.size() == 1); + if (sqlResult.get(0) == null){ + LOG.debug("getDatabase:queryDbSelector ran, returned results, but the result entry was null, returning NoSuchObjectException"); + throw new MetaException("There is no database named " + dbName); + } + + Object[] dbline = sqlResult.get(0); + Long dbid = StatObjectConverter.extractSqlLong(dbline[0]); + + String queryTextDbParams = "select \"PARAM_KEY\", \"PARAM_VALUE\" " + + " FROM \"DATABASE_PARAMS\" " + + " WHERE \"DB_ID\" = ? " + + " AND \"PARAM_KEY\" IS NOT NULL"; + Object[] params2 = new Object[] { dbid }; + queryDbParams = pm.newQuery("javax.jdo.query.SQL",queryTextDbParams); + LOG.debug("getDatabase:query2 instantiated : " + queryTextDbParams + " with param ["+params2[0]+"]"); + + Map<String,String> dbParams = new HashMap<String,String>(); + List<Object[]> sqlResult2 = ensureList(queryDbParams.executeWithArray(params2)); + if (!sqlResult2.isEmpty()){ + for (Object[] line : sqlResult2){ + dbParams.put(extractSqlString(line[0]),extractSqlString(line[1])); + } + } + LOG.debug("getDatabase: instantiating db object to return"); + Database db = new Database(); + db.setName(extractSqlString(dbline[1])); + db.setLocationUri(extractSqlString(dbline[2])); + db.setDescription(extractSqlString(dbline[3])); + db.setOwnerName(extractSqlString(dbline[4])); + String type = extractSqlString(dbline[5]); + db.setOwnerType((null == type || type.trim().isEmpty()) ? null : PrincipalType.valueOf(type)); + db.setParameters(dbParams); + if (LOG.isDebugEnabled()){ + LOG.debug("getDatabase: directsql returning db " + db.getName() + + " locn["+db.getLocationUri() +"] desc [" +db.getDescription() + + "] owner [" + db.getOwnerName() + "] ownertype ["+ db.getOwnerType() +"]"); + } + return db; + } finally { + if (queryDbSelector != null){ + queryDbSelector.closeAll(); + } + if (queryDbParams != null){ + queryDbParams.closeAll(); + } + } + } + /** * Gets partitions by using direct SQL queries. * @param dbName Metastore db name. @@ -260,10 +340,8 @@ class MetaStoreDirectSql { tblName = tblName.toLowerCase(); // We have to be mindful of order during filtering if we are not returning all partitions. String orderForFilter = (max != null) ? " order by \"PART_NAME\" asc" : ""; - if (isMySql) { - assert pm.currentTransaction().isActive(); - setAnsiQuotesForMysql(); // must be inside tx together with queries - } + + doDbSpecificInitializationsBeforeQuery(); // Get all simple fields for partitions and related objects, which we can map one-on-one. // We will do this in 2 queries to use different existing indices for each one. @@ -621,6 +699,11 @@ class MetaStoreDirectSql { return ((Number)field).intValue(); } + private String extractSqlString(Object value) { + if (value == null) return null; + return value.toString(); + } + private static String trimCommaList(StringBuilder sb) { if (sb.length() > 0) { sb.setLength(sb.length() - 1); Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java?rev=1632389&r1=1632388&r2=1632389&view=diff ============================================================================== --- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (original) +++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java Thu Oct 16 17:17:11 2014 @@ -521,6 +521,34 @@ public class ObjectStore implements RawS @Override public Database getDatabase(String name) throws NoSuchObjectException { + try { + return getDatabaseInternal(name); + } catch (MetaException e) { + // Signature restriction to NSOE, and NSOE being a flat exception prevents us from + // setting the cause of the NSOE as the MetaException. We should not lose the info + // we got here, but it's very likely that the MetaException is irrelevant and is + // actually an NSOE message, so we should log it and throw an NSOE with the msg. + LOG.warn("Got a MetaException trying to call getDatabase(" + +name+"), returning NoSuchObjectException", e); + throw new NoSuchObjectException(e.getMessage()); + } + } + + public Database getDatabaseInternal(String name) throws MetaException, NoSuchObjectException { + return new GetDbHelper(name, null, true, true) { + @Override + protected Database getSqlResult(GetHelper<Database> ctx) throws MetaException { + return directSql.getDatabase(dbName); + } + + @Override + protected Database getJdoResult(GetHelper<Database> ctx) throws MetaException, NoSuchObjectException { + return getJDODatabase(dbName); + } + }.run(false); + } + + public Database getJDODatabase(String name) throws NoSuchObjectException { MDatabase mdb = null; boolean commited = false; try { @@ -2280,7 +2308,14 @@ public class ObjectStore implements RawS assert allowSql || allowJdo; this.allowJdo = allowJdo; this.dbName = dbName.toLowerCase(); - this.tblName = tblName.toLowerCase(); + if (tblName != null){ + this.tblName = tblName.toLowerCase(); + } else { + // tblName can be null in cases of Helper being used at a higher + // abstraction level, such as with datbases + this.tblName = null; + this.table = null; + } this.doTrace = LOG.isDebugEnabled(); this.isInTxn = isActiveTransaction(); @@ -2329,7 +2364,7 @@ public class ObjectStore implements RawS private void start(boolean initTable) throws MetaException, NoSuchObjectException { start = doTrace ? System.nanoTime() : 0; openTransaction(); - if (initTable) { + if (initTable && (tblName != null)) { table = ensureGetTable(dbName, tblName); } } @@ -2396,6 +2431,27 @@ public class ObjectStore implements RawS } } + private abstract class GetDbHelper extends GetHelper<Database> { + /** + * GetHelper for returning db info using directSql/JDO. + * Since this is a db-level call, tblName is ignored, and null is passed irrespective of what is passed in. + * @param dbName The Database Name + * @param tblName Placeholder param to match signature, always ignored. + * @param allowSql Whether or not we allow DirectSQL to perform this query. + * @param allowJdo Whether or not we allow ORM to perform this query. + * @throws MetaException + */ + public GetDbHelper( + String dbName, String tblName, boolean allowSql, boolean allowJdo) throws MetaException { + super(dbName,null,allowSql,allowJdo); + } + + @Override + protected String describeResult() { + return "db details for db " + dbName; + } + } + private abstract class GetStatHelper extends GetHelper<ColumnStatistics> { public GetStatHelper( String dbName, String tblName, boolean allowSql, boolean allowJdo) throws MetaException {