Author: gates
Date: Tue Feb 10 22:47:53 2015
New Revision: 1658829
URL: http://svn.apache.org/r1658829
Log:
HIVE-9579 Support all get tables [hbase-metastore branch]
Modified:
hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreIntegration.java
hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
Modified:
hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreIntegration.java
URL:
http://svn.apache.org/viewvc/hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreIntegration.java?rev=1658829&r1=1658828&r2=1658829&view=diff
==============================================================================
---
hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreIntegration.java
(original)
+++
hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreIntegration.java
Tue Feb 10 22:47:53 2015
@@ -245,6 +245,56 @@ public class TestHBaseStoreIntegration {
}
@Test
+ public void getAllTables() throws Exception {
+ String dbNames[] = new String[]{"db0", "db1"}; // named to match getAllDbs
so we get the
+ // right number of databases in that test.
+ String tableNames[] = new String[]{"curly", "larry", "moe"};
+
+ for (int i = 0; i < dbNames.length; i++) {
+ store.createDatabase(new Database(dbNames[i], "no description",
"file:///tmp",
+ emptyParameters));
+ }
+
+ for (int i = 0; i < dbNames.length; i++) {
+ for (int j = 0; j < tableNames.length; j++) {
+ int startTime = (int) (System.currentTimeMillis() / 1000);
+ List<FieldSchema> cols = new ArrayList<FieldSchema>();
+ cols.add(new FieldSchema("col1", "int", "nocomment"));
+ SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
+ StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp",
"input", "output", false,
+ 0,
+ serde, null, null, emptyParameters);
+ Table table = new Table(tableNames[j], dbNames[i], "me", startTime,
startTime, 0, sd,
+ null,
+ emptyParameters, null, null, null);
+ store.createTable(table);
+ }
+ }
+
+ List<String> fetchedNames = store.getAllTables(dbNames[0]);
+ Assert.assertEquals(3, fetchedNames.size());
+ String[] sortedFetchedNames = fetchedNames.toArray(new
String[fetchedNames.size()]);
+ Arrays.sort(sortedFetchedNames);
+ Assert.assertArrayEquals(tableNames, sortedFetchedNames);
+
+ List<String> regexNames = store.getTables(dbNames[0], "*y");
+ Assert.assertEquals(2, regexNames.size());
+ String[] sortedRegexNames = regexNames.toArray(new
String[regexNames.size()]);
+ Arrays.sort(sortedRegexNames);
+ Assert.assertArrayEquals(Arrays.copyOfRange(tableNames, 0, 2),
sortedRegexNames);
+
+ List<Table> fetchedTables = store.getTableObjectsByName(dbNames[1],
+ Arrays.asList(Arrays.copyOfRange(tableNames, 1, 3)));
+ Assert.assertEquals(2, fetchedTables.size());
+ sortedFetchedNames = new String[fetchedTables.size()];
+ for (int i = 0; i < fetchedTables.size(); i++) {
+ sortedFetchedNames[i] = fetchedTables.get(i).getTableName();
+ }
+ Arrays.sort(sortedFetchedNames);
+ Assert.assertArrayEquals(Arrays.copyOfRange(tableNames, 1, 3),
sortedFetchedNames);
+ }
+
+ @Test
public void dropTable() throws Exception {
String tableName = "dtable";
int startTime = (int)(System.currentTimeMillis() / 1000);
Modified:
hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
URL:
http://svn.apache.org/viewvc/hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java?rev=1658829&r1=1658828&r2=1658829&view=diff
==============================================================================
---
hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
(original)
+++
hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
Tue Feb 10 22:47:53 2015
@@ -199,13 +199,10 @@ class HBaseReadWrite {
// Synchronize this so not everyone's doing it at once.
static synchronized void createTablesIfNotExist() throws IOException {
if (!tablesCreated) {
- LOG.debug("Determing which tables need created");
HBaseAdmin admin = new HBaseAdmin(self.get().conn);
- LOG.debug("Got hbase admin");
for (String name : tableNames) {
- LOG.debug("Checking for table " + name);
if (self.get().getHTable(name) == null) {
- LOG.debug("Creating table " + name);
+ LOG.info("Creating HBase table " + name);
HTableDescriptor tableDesc = new
HTableDescriptor(TableName.valueOf(name));
tableDesc.addFamily(new HColumnDescriptor(CATALOG_CF));
// Only table and partitions need stats
@@ -474,6 +471,81 @@ class HBaseReadWrite {
}
/**
+ * Fetch a list of table objects.
+ * @param dbName Database that all fetched tables are in
+ * @param tableNames list of table names
+ * @return list of tables, in the same order as the provided names.
+ * @throws IOException
+ */
+ List<Table> getTables(String dbName, List<String> tableNames) throws
IOException {
+ // I could implement getTable in terms of this method. But it is such a
core function
+ // that I don't want to slow it down for the much less common fetching of
multiple tables.
+ List<Table> results = new ArrayList<Table>(tableNames.size());
+ ObjectPair<String, String>[] hashKeys = new ObjectPair[tableNames.size()];
+ boolean atLeastOneMissing = false;
+ for (int i = 0; i < tableNames.size(); i++) {
+ hashKeys[i] = new ObjectPair<String, String>(dbName, tableNames.get(i));
+ // The result may be null, but we still want to add it so that we have a
slot in the list
+ // for it.
+ results.add(tableCache.get(hashKeys[i]));
+ if (results.get(i) == null) atLeastOneMissing = true;
+ }
+ if (!atLeastOneMissing) return results;
+
+ // Now build a single get that will fetch the remaining tables
+ List<Get> gets = new ArrayList<Get>();
+ HTableInterface htab = getHTable(TABLE_TABLE);
+ for (int i = 0; i < tableNames.size(); i++) {
+ if (results.get(i) != null) continue;
+ byte[] key = HBaseUtils.buildKey(dbName, tableNames.get(i));
+ Get g = new Get(key);
+ g.addColumn(CATALOG_CF, CATALOG_COL);
+ gets.add(g);
+ }
+ Result[] res = htab.get(gets);
+ for (int i = 0, nextGet = 0; i < tableNames.size(); i++) {
+ if (results.get(i) != null) continue;
+ byte[] serialized = res[nextGet++].getValue(CATALOG_CF, CATALOG_COL);
+ if (serialized != null) {
+ TableWritable table = new TableWritable();
+ HBaseUtils.deserialize(table, serialized);
+ tableCache.put(hashKeys[i], table.table);
+ results.set(i, table.table);
+ }
+ }
+ return results;
+ }
+
+ /**
+ * Get a list of tables.
+ * @param dbName Database these tables are in
+ * @param regex Regular expression to use in searching for table names. It
is expected to
+ * be a Java regular expression. If it is null then all tables
in the indicated
+ * database will be returned.
+ * @return list of tables matching the regular expression.
+ * @throws IOException
+ */
+ List<Table> scanTables(String dbName, String regex) throws IOException {
+ // There's no way to know whether all the tables we are looking for are
+ // in the cache, so we would need to scan one way or another. Thus
there's no value in hitting
+ // the cache for this function.
+ Filter filter = null;
+ if (regex != null) {
+ filter = new RowFilter(CompareFilter.CompareOp.EQUAL, new
RegexStringComparator(regex));
+ }
+ byte[] keyPrefix = HBaseUtils.buildKeyWithTrailingSeparator(dbName);
+ Iterator<Result> iter =
+ scanWithFilter(TABLE_TABLE, keyPrefix, CATALOG_CF, CATALOG_COL,
filter);
+ List<Table> tables = new ArrayList<Table>();
+ while (iter.hasNext()) {
+ TableWritable table = new TableWritable();
+ HBaseUtils.deserialize(table, iter.next().getValue(CATALOG_CF,
CATALOG_COL));
+ tables.add(table.table);
+ }
+ return tables;
+ }
+
+ /**
* Put a table object
* @param table table object
* @throws IOException
Modified:
hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
URL:
http://svn.apache.org/viewvc/hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java?rev=1658829&r1=1658828&r2=1658829&view=diff
==============================================================================
---
hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
(original)
+++
hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
Tue Feb 10 22:47:53 2015
@@ -322,18 +322,31 @@ public class HBaseStore implements RawSt
@Override
public List<String> getTables(String dbName, String pattern) throws
MetaException {
- throw new UnsupportedOperationException();
+ try {
+ List<Table> tables = getHBase().scanTables(dbName, likeToRegex(pattern));
+ List<String> tableNames = new ArrayList<String>(tables.size());
+ for (Table table : tables) tableNames.add(table.getTableName());
+ return tableNames;
+ } catch (IOException e) {
+ LOG.error("Unable to get tables ", e);
+ throw new MetaException("Unable to get tables, " + e.getMessage());
+ }
}
@Override
public List<Table> getTableObjectsByName(String dbname, List<String>
tableNames) throws
MetaException, UnknownDBException {
- throw new UnsupportedOperationException();
+ try {
+ return getHBase().getTables(dbname, tableNames);
+ } catch (IOException e) {
+ LOG.error("Unable to get tables ", e);
+ throw new MetaException("Unable to get tables, " + e.getMessage());
+ }
}
@Override
public List<String> getAllTables(String dbName) throws MetaException {
- throw new UnsupportedOperationException();
+ return getTables(dbName, null);
}
@Override
