Modified: hive/trunk/metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb?rev=1134183&r1=1134182&r2=1134183&view=diff ============================================================================== --- hive/trunk/metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb (original) +++ hive/trunk/metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb Fri Jun 10 03:31:08 2011 @@ -298,6 +298,24 @@ module ThriftHiveMetastore raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'get_table failed: unknown result') end + def get_table_objects_by_name(dbname, tbl_names) + send_get_table_objects_by_name(dbname, tbl_names) + return recv_get_table_objects_by_name() + end + + def send_get_table_objects_by_name(dbname, tbl_names) + send_message('get_table_objects_by_name', Get_table_objects_by_name_args, :dbname => dbname, :tbl_names => tbl_names) + end + + def recv_get_table_objects_by_name() + result = receive_message(Get_table_objects_by_name_result) + return result.success unless result.success.nil? + raise result.o1 unless result.o1.nil? + raise result.o2 unless result.o2.nil? + raise result.o3 unless result.o3.nil? + raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'get_table_objects_by_name failed: unknown result') + end + def alter_table(dbname, tbl_name, new_tbl) send_alter_table(dbname, tbl_name, new_tbl) recv_alter_table() @@ -1188,6 +1206,21 @@ module ThriftHiveMetastore write_result(result, oprot, 'get_table', seqid) end + def process_get_table_objects_by_name(seqid, iprot, oprot) + args = read_args(iprot, Get_table_objects_by_name_args) + result = Get_table_objects_by_name_result.new() + begin + result.success = @handler.get_table_objects_by_name(args.dbname, args.tbl_names) + rescue MetaException => o1 + result.o1 = o1 + rescue InvalidOperationException => o2 + result.o2 = o2 + rescue UnknownDBException => o3 + result.o3 = o3 + end + write_result(result, oprot, 'get_table_objects_by_name', seqid) + end + def process_alter_table(seqid, iprot, oprot) args = read_args(iprot, Alter_table_args) result = Alter_table_result.new() @@ -2303,6 +2336,46 @@ module ThriftHiveMetastore ::Thrift::Struct.generate_accessors self end + class Get_table_objects_by_name_args + include ::Thrift::Struct, ::Thrift::Struct_Union + DBNAME = 1 + TBL_NAMES = 2 + + FIELDS = { + DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbname'}, + TBL_NAMES => {:type => ::Thrift::Types::LIST, :name => 'tbl_names', :element => {:type => ::Thrift::Types::STRING}} + } + + def struct_fields; FIELDS; end + + def validate + end + + ::Thrift::Struct.generate_accessors self + end + + class Get_table_objects_by_name_result + include ::Thrift::Struct, ::Thrift::Struct_Union + SUCCESS = 0 + O1 = 1 + O2 = 2 + O3 = 3 + + FIELDS = { + SUCCESS => {:type => ::Thrift::Types::LIST, :name => 'success', :element => {:type => ::Thrift::Types::STRUCT, :class => Table}}, + O1 => {:type => ::Thrift::Types::STRUCT, :name => 'o1', :class => MetaException}, + O2 => {:type => ::Thrift::Types::STRUCT, :name => 'o2', :class => InvalidOperationException}, + O3 => {:type => ::Thrift::Types::STRUCT, :name => 'o3', :class => UnknownDBException} + } + + def struct_fields; FIELDS; end + + def validate + end + + ::Thrift::Struct.generate_accessors self + end + class Alter_table_args include ::Thrift::Struct, ::Thrift::Struct_Union DBNAME = 1
Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java?rev=1134183&r1=1134182&r2=1134183&view=diff ============================================================================== --- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (original) +++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java Fri Jun 10 03:31:08 2011 @@ -516,6 +516,11 @@ public class HiveMetaStore extends Thrif return startFunction(function, " : db=" + db + " tbl=" + tbl); } + public String startMultiTableFunction(String function, String db, List<String> tbls) { + String tableNames = join(tbls, ","); + return startFunction(function, " : db=" + db + " tbls=" + tableNames); + } + public String startPartitionFunction(String function, String db, String tbl, List<String> partVals) { return startFunction(function, " : db=" + db + " tbl=" + tbl @@ -1134,6 +1139,54 @@ public class HiveMetaStore extends Thrif return t; } + /** + * Gets multiple tables from the hive metastore. + * @param dbname + * The name of the database in which the tables reside + * @param names + * The names of the tables to get. + * + * @return A list of tables whose names are in the the list "names" and + * are retrievable from the database specified by "dbnames." + * There is no guarantee of the order of the returned tables. + * If there are duplicate names, only one instance of the table will be returned. + * @throws MetaException + * @throws InvalidOperationException + * @throws UnknownDBException + */ + public List<Table> get_table_objects_by_name(final String dbname, final List<String> names) + throws MetaException, InvalidOperationException, UnknownDBException { + List<Table> tables = new ArrayList<Table>(); + startMultiTableFunction("get_multi_table", dbname, names); + try { + tables = executeWithRetry(new Command<List<Table>>() { + @Override + public List<Table> run(RawStore ms) throws Exception { + if (dbname == null || dbname.isEmpty()) { + throw new UnknownDBException("DB name is null or empty"); + } + if (names == null) + { + throw new InvalidOperationException(dbname + " cannot find null tables"); + } + List<Table> foundTables = ms.getTableObjectsByName(dbname, names); + return foundTables; + } + }); + } catch (MetaException e) { + throw e; + } catch (InvalidOperationException e) { + throw e; + } catch (UnknownDBException e) { + throw e; + } catch (Exception e) { + throw new MetaException(e.toString()); + } finally { + endFunction("get_multi_table"); + } + return tables; + } + public boolean set_table_parameters(String dbname, String name, Map<String, String> params) throws NoSuchObjectException, MetaException { endFunction(startTableFunction("set_table_parameters", dbname, name)); Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java?rev=1134183&r1=1134182&r2=1134183&view=diff ============================================================================== --- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java (original) +++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java Fri Jun 10 03:31:08 2011 @@ -688,6 +688,12 @@ public class HiveMetaStoreClient impleme return getTable(DEFAULT_DATABASE_NAME, tableName); } + /** {@inheritDoc} */ + public List<Table> getTableObjectsByName(String dbName, List<String> tableNames) + throws MetaException, InvalidOperationException, UnknownDBException, TException { + return deepCopyTables(client.get_table_objects_by_name(dbName, tableNames)); + } + /** * @param name * @return the type @@ -1064,7 +1070,7 @@ public class HiveMetaStoreClient impleme String owner = conf.getUser(); return getDelegationToken(owner, renewerKerberosPrincipalName); } - + @Override public String getDelegationToken(String owner, String renewerKerberosPrincipalName) throws MetaException, TException { Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java?rev=1134183&r1=1134182&r2=1134183&view=diff ============================================================================== --- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java (original) +++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java Fri Jun 10 03:31:08 2011 @@ -207,6 +207,28 @@ public interface IMetaStoreClient { TException, NoSuchObjectException; /** + * + * @param dbName + * The database the tables are located in. + * @param tableNames + * The names of the tables to fetch + * @return A list of objects representing the tables. + * Only the tables that can be retrieved from the database are returned. For example, + * if none of the requested tables could be retrieved, an empty list is returned. + * There is no guarantee of ordering of the returned tables. + * @throws InvalidOperationException + * The input to this operation is invalid (e.g., the list of tables names is null) + * @throws UnknownDBException + * The requested database could not be fetched. + * @throws TException + * A thrift communication error occurred + * @throws MetaException + * Any other errors + */ + public List<Table> getTableObjectsByName(String dbName, List<String> tableNames) + throws MetaException, InvalidOperationException, UnknownDBException, TException; + + /** * @param tableName * @param dbName * @param partVals Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java?rev=1134183&r1=1134182&r2=1134183&view=diff ============================================================================== --- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (original) +++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java Fri Jun 10 03:31:08 2011 @@ -26,9 +26,9 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Properties; import java.util.Set; -import java.util.Map.Entry; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@ -71,6 +71,7 @@ import org.apache.hadoop.hive.metastore. import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.Type; +import org.apache.hadoop.hive.metastore.api.UnknownDBException; import org.apache.hadoop.hive.metastore.model.MDBPrivilege; import org.apache.hadoop.hive.metastore.model.MDatabase; import org.apache.hadoop.hive.metastore.model.MFieldSchema; @@ -88,9 +89,9 @@ import org.apache.hadoop.hive.metastore. import org.apache.hadoop.hive.metastore.model.MTableColumnPrivilege; import org.apache.hadoop.hive.metastore.model.MTablePrivilege; import org.apache.hadoop.hive.metastore.model.MType; +import org.apache.hadoop.hive.metastore.parser.ExpressionTree.ANTLRNoCaseStringStream; import org.apache.hadoop.hive.metastore.parser.FilterLexer; import org.apache.hadoop.hive.metastore.parser.FilterParser; -import org.apache.hadoop.hive.metastore.parser.ExpressionTree.ANTLRNoCaseStringStream; import org.apache.hadoop.util.StringUtils; /** @@ -787,6 +788,43 @@ public class ObjectStore implements RawS return mtbl; } + public List<Table> getTableObjectsByName(String db, List<String> tbl_names) + throws MetaException, UnknownDBException { + List<Table> tables = new ArrayList<Table>(); + boolean committed = false; + try { + openTransaction(); + + db = db.toLowerCase().trim(); + Query dbExistsQuery = pm.newQuery(MDatabase.class, "name == db"); + dbExistsQuery.declareParameters("java.lang.String db"); + dbExistsQuery.setUnique(true); + dbExistsQuery.setResult("name"); + String dbNameIfExists = (String) dbExistsQuery.execute(db); + if (dbNameIfExists == null || dbNameIfExists.isEmpty()) { + throw new UnknownDBException("Could not find database " + db); + } + + List<String> lowered_tbl_names = new ArrayList<String>(); + for (String t : tbl_names) { + lowered_tbl_names.add(t.toLowerCase().trim()); + } + Query query = pm.newQuery(MTable.class); + query.setFilter("database.name == db && tbl_names.contains(tableName)"); + query.declareParameters("java.lang.String db, java.util.Collection tbl_names"); + Collection mtables = (Collection) query.execute(db, lowered_tbl_names); + for (Iterator iter = mtables.iterator(); iter.hasNext();) { + tables.add(convertToTable((MTable) iter.next())); + } + committed = commitTransaction(); + } finally { + if (!committed) { + rollbackTransaction(); + } + } + return tables; + } + private Table convertToTable(MTable mtbl) throws MetaException { if (mtbl == null) { return null; Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java?rev=1134183&r1=1134182&r2=1134183&view=diff ============================================================================== --- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java (original) +++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java Fri Jun 10 03:31:08 2011 @@ -33,6 +33,7 @@ import org.apache.hadoop.hive.metastore. import org.apache.hadoop.hive.metastore.api.Role; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.Type; +import org.apache.hadoop.hive.metastore.api.UnknownDBException; import org.apache.hadoop.hive.metastore.model.MDBPrivilege; import org.apache.hadoop.hive.metastore.model.MGlobalPrivilege; import org.apache.hadoop.hive.metastore.model.MPartitionColumnPrivilege; @@ -114,6 +115,19 @@ public interface RawStore extends Config public List<String> getTables(String dbName, String pattern) throws MetaException; + /** + * @param dbname + * The name of the database from which to retrieve the tables + * @param tableNames + * The names of the tables to retrieve. + * @return A list of the tables retrievable from the database + * whose names are in the list tableNames. + * If there are duplicate names, only one instance of the table will be returned + * @throws MetaException + */ + public List<Table> getTableObjectsByName(String dbname, List<String> tableNames) + throws MetaException, UnknownDBException; + public List<String> getAllTables(String dbName) throws MetaException; public abstract List<String> listPartitionNames(String db_name, Modified: hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java?rev=1134183&r1=1134182&r2=1134183&view=diff ============================================================================== --- hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java (original) +++ hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java Fri Jun 10 03:31:08 2011 @@ -45,6 +45,7 @@ import org.apache.hadoop.hive.metastore. import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.Type; +import org.apache.hadoop.hive.metastore.api.UnknownDBException; import org.apache.hadoop.hive.serde.Constants; import org.apache.hadoop.util.StringUtils; import org.apache.thrift.TException; @@ -716,6 +717,59 @@ public abstract class TestHiveMetaStore (tbl2.getPartitionKeys() == null) || (tbl2.getPartitionKeys().size() == 0)); + //test get_table_objects_by_name functionality + ArrayList<String> tableNames = new ArrayList<String>(); + tableNames.add(tblName2); + tableNames.add(tblName); + tableNames.add(tblName2); + List<Table> foundTables = client.getTableObjectsByName(dbName, tableNames); + + assertEquals(foundTables.size(), 2); + for (Table t: foundTables) { + if (t.getTableName().equals(tblName2)) { + assertEquals(t.getSd().getLocation(), tbl2.getSd().getLocation()); + } else { + assertEquals(t.getTableName(), tblName); + assertEquals(t.getSd().getLocation(), tbl.getSd().getLocation()); + } + assertEquals(t.getSd().getCols().size(), typ1.getFields().size()); + assertEquals(t.getSd().isCompressed(), false); + assertEquals(foundTables.get(0).getSd().getNumBuckets(), 1); + assertNotNull(t.getSd().getSerdeInfo()); + assertEquals(t.getDbName(), dbName); + } + + tableNames.add(1, "table_that_doesnt_exist"); + foundTables = client.getTableObjectsByName(dbName, tableNames); + assertEquals(foundTables.size(), 2); + + InvalidOperationException ioe = null; + try { + foundTables = client.getTableObjectsByName(dbName, null); + } catch (InvalidOperationException e) { + ioe = e; + } + assertNotNull(ioe); + assertTrue("Table not found", ioe.getMessage().contains("null tables")); + + UnknownDBException udbe = null; + try { + foundTables = client.getTableObjectsByName("db_that_doesnt_exist", tableNames); + } catch (UnknownDBException e) { + udbe = e; + } + assertNotNull(udbe); + assertTrue("DB not found", udbe.getMessage().contains("not find database db_that_doesnt_exist")); + + udbe = null; + try { + foundTables = client.getTableObjectsByName("", tableNames); + } catch (UnknownDBException e) { + udbe = e; + } + assertNotNull(udbe); + assertTrue("DB not found", udbe.getMessage().contains("is null or empty")); + FileSystem fs = FileSystem.get((new Path(tbl.getSd().getLocation())).toUri(), hiveConf); client.dropTable(dbName, tblName); assertFalse(fs.exists(new Path(tbl.getSd().getLocation())));
