Author: gates
Date: Tue Feb 10 21:41:19 2015
New Revision: 1658821
URL: http://svn.apache.org/r1658821
Log:
HIVE-9578 Add support for getDatabases and alterDatabase calls [hbase-metastore
branch]
Modified:
hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreIntegration.java
hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
hive/branches/hbase-metastore/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStore.java
Modified:
hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreIntegration.java
URL:
http://svn.apache.org/viewvc/hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreIntegration.java?rev=1658821&r1=1658820&r2=1658821&view=diff
==============================================================================
---
hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreIntegration.java
(original)
+++
hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreIntegration.java
Tue Feb 10 21:41:19 2015
@@ -20,7 +20,6 @@ package org.apache.hadoop.hive.metastore
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.client.HConnection;
import org.apache.hadoop.hbase.client.HTableInterface;
@@ -148,6 +147,44 @@ public class TestHBaseStoreIntegration {
}
@Test
+ public void getAllDbs() throws Exception {
+ String[] dbNames = new String[3];
+ for (int i = 0; i < dbNames.length; i++) {
+ dbNames[i] = "db" + i;
+ Database db = new Database(dbNames[i], "no description", "file:///tmp",
emptyParameters);
+ store.createDatabase(db);
+ }
+
+ List<String> dbs = store.getAllDatabases();
+ Assert.assertEquals(3, dbs.size());
+ String[] namesFromStore = dbs.toArray(new String[3]);
+ Arrays.sort(namesFromStore);
+ Assert.assertArrayEquals(dbNames, namesFromStore);
+ }
+
+ @Test
+ public void getDbsRegex() throws Exception {
+ String[] dbNames = new String[3];
+ for (int i = 0; i < dbNames.length; i++) {
+ dbNames[i] = "db" + i;
+ Database db = new Database(dbNames[i], "no description", "file:///tmp",
emptyParameters);
+ store.createDatabase(db);
+ }
+
+ List<String> dbs = store.getDatabases("db1|db2");
+ Assert.assertEquals(2, dbs.size());
+ String[] namesFromStore = dbs.toArray(new String[2]);
+ Arrays.sort(namesFromStore);
+ Assert.assertArrayEquals(Arrays.copyOfRange(dbNames, 1, 3),
namesFromStore);
+
+ dbs = store.getDatabases("db*");
+ Assert.assertEquals(3, dbs.size());
+ namesFromStore = dbs.toArray(new String[3]);
+ Arrays.sort(namesFromStore);
+ Assert.assertArrayEquals(dbNames, namesFromStore);
+ }
+
+ @Test
public void createTable() throws Exception {
int startTime = (int)(System.currentTimeMillis() / 1000);
List<FieldSchema> cols = new ArrayList<FieldSchema>();
@@ -432,7 +469,7 @@ public class TestHBaseStoreIntegration {
@Test
public void createRole() throws Exception {
- int now = (int)System.currentTimeMillis();
+ int now = (int)System.currentTimeMillis()/1000;
String roleName = "myrole";
store.addRole(roleName, "me");
@@ -444,14 +481,11 @@ public class TestHBaseStoreIntegration {
@Test
public void dropRole() throws Exception {
- int now = (int)System.currentTimeMillis();
String roleName = "anotherrole";
store.addRole(roleName, "me");
Role r = store.getRole(roleName);
Assert.assertEquals(roleName, r.getRoleName());
- Assert.assertEquals("me", r.getOwnerName());
- Assert.assertTrue(now <= r.getCreateTime());
store.removeRole(roleName);
thrown.expect(NoSuchObjectException.class);
Modified:
hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
URL:
http://svn.apache.org/viewvc/hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java?rev=1658821&r1=1658820&r2=1658821&view=diff
==============================================================================
---
hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
(original)
+++
hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
Tue Feb 10 21:41:19 2015
@@ -259,6 +259,29 @@ class HBaseReadWrite {
}
/**
+ * Get a list of databases.
+ * @param regex Regular expression to use in searching for database names.
It is expected to
+ * be a Java regular expression. If it is null then all
databases will be returned.
+ * @return list of databases matching the regular expression.
+ * @throws IOException
+ */
+ List<Database> scanDatabases(String regex) throws IOException {
+ Filter filter = null;
+ if (regex != null) {
+ filter = new RowFilter(CompareFilter.CompareOp.EQUAL, new
RegexStringComparator(regex));
+ }
+ Iterator<Result> iter =
+ scanWithFilter(DB_TABLE, null, CATALOG_CF, CATALOG_COL, filter);
+ List<Database> databases = new ArrayList<Database>();
+ while (iter.hasNext()) {
+ DatabaseWritable db = new DatabaseWritable();
+ HBaseUtils.deserialize(db, iter.next().getValue(CATALOG_CF,
CATALOG_COL));
+ databases.add(db.db);
+ }
+ return databases;
+ }
+
+ /**
* Store a database object
* @param database database object to store
* @throws IOException
@@ -328,7 +351,7 @@ class HBaseReadWrite {
: new ArrayList<Partition>(cached);
}
byte[] keyPrefix = HBaseUtils.buildKeyWithTrailingSeparator(dbName,
tableName);
- List<Partition> parts = scanOnPrefix(PART_TABLE, keyPrefix, CATALOG_CF,
CATALOG_COL, -1);
+ List<Partition> parts = scanPartitions(keyPrefix, CATALOG_CF, CATALOG_COL,
-1);
partCache.put(dbName, tableName, parts, true);
return maxPartitions < parts.size() ? parts.subList(0, maxPartitions) :
parts;
}
@@ -352,7 +375,7 @@ class HBaseReadWrite {
byte[] keyPrefix;
if (partVals == null || partVals.size() == 0) {
keyPrefix = HBaseUtils.buildKeyWithTrailingSeparator(dbName, tableName);
- return scanOnPrefix(PART_TABLE, keyPrefix, CATALOG_CF, CATALOG_COL,
maxPartitions);
+ return scanPartitions(keyPrefix, CATALOG_CF, CATALOG_COL, maxPartitions);
}
int firstNull = 0;
for (; firstNull < partVals.size(); firstNull++) {
@@ -360,7 +383,7 @@ class HBaseReadWrite {
}
if (firstNull == partVals.size()) {
keyPrefix = buildPartitionKey(dbName, tableName, partVals);
- return scanOnPrefix(PART_TABLE, keyPrefix, CATALOG_CF, CATALOG_COL,
maxPartitions);
+ return scanPartitions(keyPrefix, CATALOG_CF, CATALOG_COL, maxPartitions);
}
keyPrefix = buildPartitionKey(dbName, tableName, partVals.subList(0,
firstNull));
StringBuilder regex = new StringBuilder();
@@ -377,8 +400,8 @@ class HBaseReadWrite {
Filter filter = new RowFilter(CompareFilter.CompareOp.EQUAL,
new RegexStringComparator(regex.toString()));
- List<Partition> parts = scanOnPrefixWithFilter(PART_TABLE, keyPrefix,
CATALOG_CF, CATALOG_COL,
- maxPartitions, filter);
+ List<Partition> parts =
+ scanPartitionsWithFilter(keyPrefix, CATALOG_CF, CATALOG_COL,
maxPartitions, filter);
partCache.put(dbName, tableName, parts, false);
return parts;
}
@@ -835,24 +858,18 @@ class HBaseReadWrite {
htab.delete(d);
}
- private List<Partition> scanOnPrefix(String table, byte[] keyPrefix, byte[]
colFam, byte[] colName,
- int maxResults) throws IOException {
- return scanOnPrefixWithFilter(table, keyPrefix, colFam, colName,
maxResults, null);
+ private List<Partition> scanPartitions(byte[] keyPrefix, byte[] colFam,
byte[] colName,
+ int maxResults) throws IOException {
+ return scanPartitionsWithFilter(keyPrefix, colFam, colName, maxResults,
null);
}
- private List<Partition> scanOnPrefixWithFilter(String table, byte[]
keyPrefix, byte[] colFam,
- byte[] colName, int
maxResults, Filter filter)
+ private List<Partition> scanPartitionsWithFilter(byte[] keyPrefix, byte[]
colFam, byte[] colName,
+ int maxResults, Filter
filter)
throws IOException {
- HTableInterface htab = getHTable(table);
- byte[] stop = Arrays.copyOf(keyPrefix, keyPrefix.length);
- stop[stop.length - 1]++;
- Scan s = new Scan(keyPrefix, stop);
- s.addColumn(colFam, colName);
- if (filter != null) s.setFilter(filter);
- ResultScanner scanner = htab.getScanner(s);
+ Iterator<Result> iter =
+ scanWithFilter(PART_TABLE, keyPrefix, colFam, colName, filter);
List<Partition> parts = new ArrayList<Partition>();
int numToFetch = maxResults < 0 ? Integer.MAX_VALUE : maxResults;
- Iterator<Result> iter = scanner.iterator();
for (int i = 0; i < numToFetch && iter.hasNext(); i++) {
PartitionWritable p = new PartitionWritable();
HBaseUtils.deserialize(p, iter.next().getValue(colFam, colName));
@@ -861,6 +878,23 @@ class HBaseReadWrite {
return parts;
}
+ private Iterator<Result> scanWithFilter(String table, byte[] keyPrefix,
byte[] colFam,
+ byte[] colName, Filter filter)
throws IOException {
+ HTableInterface htab = getHTable(table);
+ Scan s;
+ if (keyPrefix == null) {
+ s = new Scan();
+ } else {
+ byte[] stop = Arrays.copyOf(keyPrefix, keyPrefix.length);
+ stop[stop.length - 1]++;
+ s = new Scan(keyPrefix, stop);
+ }
+ s.addColumn(colFam, colName);
+ if (filter != null) s.setFilter(filter);
+ ResultScanner scanner = htab.getScanner(s);
+ return scanner.iterator();
+ }
+
private HTableInterface getHTable(String table) throws IOException {
HTableInterface htab = tables.get(table);
if (htab == null) {
Modified:
hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
URL:
http://svn.apache.org/viewvc/hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java?rev=1658821&r1=1658820&r2=1658821&view=diff
==============================================================================
---
hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
(original)
+++
hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
Tue Feb 10 21:41:19 2015
@@ -148,17 +148,33 @@ public class HBaseStore implements RawSt
@Override
public boolean alterDatabase(String dbname, Database db) throws
NoSuchObjectException,
MetaException {
- throw new UnsupportedOperationException();
+ // ObjectStore fetches the old db before updating it, but I can't see the
possible value of
+ // that since the caller will have needed to call getDatabase to have the
db object.
+ try {
+ getHBase().putDb(db);
+ return true;
+ } catch (IOException e) {
+ LOG.error("Unable to alter database ", e);
+ throw new MetaException("Unable to read from or write to hbase " +
e.getMessage());
+ }
}
@Override
public List<String> getDatabases(String pattern) throws MetaException {
- throw new UnsupportedOperationException();
+ try {
+ List<Database> dbs = getHBase().scanDatabases(likeToRegex(pattern));
+ List<String> dbNames = new ArrayList<String>(dbs.size());
+ for (Database db : dbs) dbNames.add(db.getName());
+ return dbNames;
+ } catch (IOException e) {
+ LOG.error("Unable to get databases ", e);
+ throw new MetaException("Unable to get databases, " + e.getMessage());
+ }
}
@Override
public List<String> getAllDatabases() throws MetaException {
- throw new UnsupportedOperationException();
+ return getDatabases(null);
}
@Override
@@ -964,4 +980,14 @@ public class HBaseStore implements RawSt
}
return vals;
}
+
+ private String likeToRegex(String like) {
+ if (like == null) return null;
+ // Convert Hive's strange like syntax to Java regex. Per
+ //
https://cwiki.apache.org/confluence/display/Hive/LanguageManual+DDL#LanguageManualDDL-Show
+ // the supported syntax is that * means Java .* and | means 'or'
+ // This implementation leaves other regular expression syntax alone, which
means people can
+ // use it, even though it wouldn't work on RDBMS backed metastores.
+ return like.replace("*", ".*");
+ }
}
Modified:
hive/branches/hbase-metastore/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStore.java
URL:
http://svn.apache.org/viewvc/hive/branches/hbase-metastore/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStore.java?rev=1658821&r1=1658820&r2=1658821&view=diff
==============================================================================
---
hive/branches/hbase-metastore/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStore.java
(original)
+++
hive/branches/hbase-metastore/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStore.java
Tue Feb 10 21:41:19 2015
@@ -105,6 +105,20 @@ public class TestHBaseStore {
}
@Test
+ public void alterDb() throws Exception {
+ String dbname = "mydb";
+ Database db = new Database(dbname, "no description", "file:///tmp",
emptyParameters);
+ store.createDatabase(db);
+ db.setDescription("a description");
+ store.alterDatabase(dbname, db);
+
+ Database d = store.getDatabase(dbname);
+ Assert.assertEquals(dbname, d.getName());
+ Assert.assertEquals("a description", d.getDescription());
+ Assert.assertEquals("file:///tmp", d.getLocationUri());
+ }
+
+ @Test
public void dropDb() throws Exception {
String dbname = "anotherdb";
Database db = new Database(dbname, "no description", "file:///tmp",
emptyParameters);
@@ -352,7 +366,7 @@ public class TestHBaseStore {
@Test
public void createRole() throws Exception {
- int now = (int)System.currentTimeMillis();
+ int now = (int)System.currentTimeMillis()/1000;
String roleName = "myrole";
store.addRole(roleName, "me");
