Author: namit
Date: Fri Dec 2 05:55:26 2011
New Revision: 1209368
URL: http://svn.apache.org/viewvc?rev=1209368&view=rev
Log:
HIVE-2619 Add hook to run in metastore's endFunction which can collect more
fb303 counters
(Kevin Wilfong via namit)
Added:
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionContext.java
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionListener.java
Modified:
hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/trunk/conf/hive-default.xml
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL:
http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1209368&r1=1209367&r2=1209368&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
(original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Fri
Dec 2 05:55:26 2011
@@ -92,6 +92,7 @@ public class HiveConf extends Configurat
HiveConf.ConfVars.METASTORE_EVENT_CLEAN_FREQ,
HiveConf.ConfVars.METASTORE_EVENT_EXPIRY_DURATION,
HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL,
+ HiveConf.ConfVars.METASTORE_END_FUNCTION_LISTENERS,
};
/**
@@ -245,6 +246,7 @@ public class HiveConf extends Configurat
METASTORE_DETACH_ALL_ON_COMMIT("javax.jdo.option.DetachAllOnCommit", true),
METASTORE_NON_TRANSACTIONAL_READ("javax.jdo.option.NonTransactionalRead",
true),
METASTORE_CONNECTION_USER_NAME("javax.jdo.option.ConnectionUserName",
"APP"),
+ METASTORE_END_FUNCTION_LISTENERS("hive.metastore.end.function.listeners",
""),
// CLI
CLIIGNOREERRORS("hive.cli.errors.ignore", false),
Modified: hive/trunk/conf/hive-default.xml
URL:
http://svn.apache.org/viewvc/hive/trunk/conf/hive-default.xml?rev=1209368&r1=1209367&r2=1209368&view=diff
==============================================================================
--- hive/trunk/conf/hive-default.xml (original)
+++ hive/trunk/conf/hive-default.xml Fri Dec 2 05:55:26 2011
@@ -239,6 +239,12 @@
</property>
<property>
+ <name>hive.metastore.end.function.listeners</name>
+ <value></value>
+ <description>list of comma separated listeners for the end of metastore
functions.</description>
+</property>
+
+<property>
<name>hive.metastore.event.expiry.duration</name>
<value>0</value>
<description>Duration after which events expire from events table (in
seconds)</description>
Modified:
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
URL:
http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java?rev=1209368&r1=1209367&r2=1209368&view=diff
==============================================================================
---
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
(original)
+++
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
Fri Dec 2 05:55:26 2011
@@ -24,6 +24,7 @@ import static org.apache.hadoop.hive.met
import static org.apache.hadoop.hive.metastore.MetaStoreUtils.validateName;
import java.io.IOException;
+import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Formatter;
@@ -230,6 +231,7 @@ public class HiveMetaStore extends Thrif
private ClassLoader classLoader;
private AlterHandler alterHandler;
private List<MetaStoreEventListener> listeners;
+ private List<MetaStoreEndFunctionListener> endFunctionListeners;
{
classLoader = Thread.currentThread().getContextClassLoader();
@@ -270,7 +272,13 @@ public class HiveMetaStore extends Thrif
}
}
- listeners = MetaStoreUtils.getMetaStoreListener(hiveConf);
+
+ listeners =
MetaStoreUtils.getMetaStoreListeners(MetaStoreEventListener.class, hiveConf,
+ hiveConf.getVar(HiveConf.ConfVars.METASTORE_EVENT_LISTENERS));
+ endFunctionListeners = MetaStoreUtils.getMetaStoreListeners(
+ MetaStoreEndFunctionListener.class, hiveConf,
+ hiveConf.getVar(HiveConf.ConfVars.METASTORE_END_FUNCTION_LISTENERS));
+
long cleanFreq =
hiveConf.getLongVar(ConfVars.METASTORE_EVENT_CLEAN_FREQ) * 1000L;
if(cleanFreq > 0){
// In default config, there is no timer.
@@ -558,12 +566,20 @@ public class HiveMetaStore extends Thrif
return startFunction(function, " : db=" + db + " tbl=" + tbl +
"partition=" + partName);
}
- public void endFunction(String function) {
+ public void endFunction(String function, boolean successful) {
+ endFunction(function, new MetaStoreEndFunctionContext(successful));
+ }
+
+ public void endFunction(String function, MetaStoreEndFunctionContext
context) {
try {
Metrics.endScope(function);
} catch (IOException e) {
LOG.debug("Exception when closing metrics scope" + e);
}
+
+ for (MetaStoreEndFunctionListener listener : endFunctionListeners) {
+ listener.onEndFunction(function, context);
+ }
}
@Override
@@ -582,6 +598,20 @@ public class HiveMetaStore extends Thrif
logInfo("Metastore shutdown complete.");
}
+ @Override
+ public AbstractMap<String, Long> getCounters() {
+ AbstractMap<String, Long> counters = super.getCounters();
+
+ // Allow endFunctionListeners to add any counters they have collected
+ if (endFunctionListeners != null) {
+ for (MetaStoreEndFunctionListener listener : endFunctionListeners) {
+ listener.exportCounters(counters);
+ }
+ }
+
+ return counters;
+ }
+
private static final String DATABASE_WAREHOUSE_SUFFIX = ".db";
private Path getDefaultDatabasePath(String dbName) throws MetaException {
@@ -636,6 +666,7 @@ public class HiveMetaStore extends Thrif
+ db.getName() + " "
+ db.getLocationUri() + " "
+ db.getDescription());
+ boolean success = false;
try {
try {
if(null != get_database(db.getName())) {
@@ -644,7 +675,7 @@ public class HiveMetaStore extends Thrif
} catch (NoSuchObjectException e) {
// expected
}
- executeWithRetry(new Command<Boolean>() {
+ success = executeWithRetry(new Command<Boolean>() {
@Override
public Boolean run(RawStore ms) throws Exception {
create_database_core(ms, db);
@@ -661,7 +692,7 @@ public class HiveMetaStore extends Thrif
assert(e instanceof RuntimeException);
throw (RuntimeException)e;
} finally {
- endFunction("create_database");
+ endFunction("create_database", success);
}
}
@@ -684,7 +715,7 @@ public class HiveMetaStore extends Thrif
assert(e instanceof RuntimeException);
throw (RuntimeException)e;
} finally {
- endFunction("get_database");
+ endFunction("get_database", db != null);
}
return db;
}
@@ -692,8 +723,9 @@ public class HiveMetaStore extends Thrif
public void alter_database(final String dbName, final Database db)
throws NoSuchObjectException, TException, MetaException {
startFunction("alter_database" + dbName);
+ boolean success = false;
try {
- executeWithRetry(new Command<Boolean>() {
+ success = executeWithRetry(new Command<Boolean>() {
@Override
public Boolean run(RawStore ms) throws Exception {
return ms.alterDatabase(dbName, db);
@@ -709,7 +741,7 @@ public class HiveMetaStore extends Thrif
assert(e instanceof RuntimeException);
throw (RuntimeException) e;
} finally {
- endFunction("alter_database");
+ endFunction("alter_database", success);
}
}
@@ -753,12 +785,13 @@ public class HiveMetaStore extends Thrif
startFunction("drop_database", ": " + dbName);
if (DEFAULT_DATABASE_NAME.equalsIgnoreCase(dbName)) {
- endFunction("drop_database");
+ endFunction("drop_database", false);
throw new MetaException("Can not drop default database");
}
+ boolean success = false;
try {
- executeWithRetry(new Command<Boolean>() {
+ success = executeWithRetry(new Command<Boolean>() {
@Override
public Boolean run(RawStore ms) throws Exception {
drop_database_core(ms, dbName, deleteData, cascade);
@@ -775,7 +808,7 @@ public class HiveMetaStore extends Thrif
assert(e instanceof RuntimeException);
throw (RuntimeException)e;
} finally {
- endFunction("drop_database");
+ endFunction("drop_database", success);
}
}
@@ -796,7 +829,7 @@ public class HiveMetaStore extends Thrif
assert(e instanceof RuntimeException);
throw (RuntimeException)e;
} finally {
- endFunction("get_databases");
+ endFunction("get_databases", ret != null);
}
return ret;
}
@@ -818,7 +851,7 @@ public class HiveMetaStore extends Thrif
assert(e instanceof RuntimeException);
throw (RuntimeException)e;
} finally {
- endFunction("get_all_databases");
+ endFunction("get_all_databases", ret != null);
}
return ret;
}
@@ -847,7 +880,7 @@ public class HiveMetaStore extends Thrif
public boolean create_type(final Type type) throws AlreadyExistsException,
MetaException, InvalidObjectException {
startFunction("create_type", ": " + type.getName());
- Boolean ret = null;
+ boolean ret = false;
try {
ret = executeWithRetry(new Command<Boolean>() {
@Override
@@ -866,16 +899,16 @@ public class HiveMetaStore extends Thrif
assert(e instanceof RuntimeException);
throw (RuntimeException)e;
} finally {
- endFunction("create_type");
+ endFunction("create_type", ret);
}
- return ret.booleanValue();
+ return ret;
}
public Type get_type(final String name) throws MetaException,
NoSuchObjectException {
startFunction("get_type", ": " + name);
- Type ret;
+ Type ret = null;
try {
ret = executeWithRetry(new Command<Type>() {
@Override
@@ -895,7 +928,7 @@ public class HiveMetaStore extends Thrif
assert(e instanceof RuntimeException);
throw (RuntimeException)e;
} finally {
- endFunction("get_type");
+ endFunction("get_type", ret != null);
}
return ret;
}
@@ -929,7 +962,7 @@ public class HiveMetaStore extends Thrif
public boolean drop_type(final String name) throws MetaException {
startFunction("drop_type", ": " + name);
- Boolean ret = null;
+ boolean ret = false;
try {
ret = executeWithRetry(new Command<Boolean>() {
@Override
@@ -944,7 +977,7 @@ public class HiveMetaStore extends Thrif
assert(e instanceof RuntimeException);
throw (RuntimeException)e;
} finally {
- endFunction("drop_type");
+ endFunction("drop_type", ret);
}
return ret;
}
@@ -952,7 +985,7 @@ public class HiveMetaStore extends Thrif
public Map<String, Type> get_type_all(String name) throws MetaException {
// TODO Auto-generated method stub
startFunction("get_type_all", ": " + name);
- endFunction("get_type_all");
+ endFunction("get_type_all", false);
throw new MetaException("Not yet implemented");
}
@@ -1034,8 +1067,9 @@ public class HiveMetaStore extends Thrif
MetaException, InvalidObjectException {
startFunction("create_table", ": db=" + tbl.getDbName() + " tbl="
+ tbl.getTableName());
+ boolean success = false;
try {
- executeWithRetry(new Command<Boolean>() {
+ success = executeWithRetry(new Command<Boolean>() {
@Override
public Boolean run(RawStore ms) throws Exception {
create_table_core(ms, tbl);
@@ -1054,7 +1088,7 @@ public class HiveMetaStore extends Thrif
assert(e instanceof RuntimeException);
throw (RuntimeException)e;
} finally {
- endFunction("create_table");
+ endFunction("create_table", success);
}
}
@@ -1134,8 +1168,9 @@ public class HiveMetaStore extends Thrif
throws NoSuchObjectException, MetaException {
startTableFunction("drop_table", dbname, name);
+ boolean success = false;
try {
- executeWithRetry(new Command<Boolean>() {
+ success = executeWithRetry(new Command<Boolean>() {
@Override
public Boolean run(RawStore ms) throws Exception {
drop_table_core(ms, dbname, name, deleteData);
@@ -1150,7 +1185,7 @@ public class HiveMetaStore extends Thrif
assert(e instanceof RuntimeException);
throw (RuntimeException)e;
} finally {
- endFunction("drop_table");
+ endFunction("drop_table", success);
}
}
@@ -1194,7 +1229,7 @@ public class HiveMetaStore extends Thrif
assert(e instanceof RuntimeException);
throw (RuntimeException)e;
} finally {
- endFunction("get_table");
+ endFunction("get_table", t != null);
}
return t;
}
@@ -1216,7 +1251,7 @@ public class HiveMetaStore extends Thrif
*/
public List<Table> get_table_objects_by_name(final String dbname, final
List<String> names)
throws MetaException, InvalidOperationException, UnknownDBException {
- List<Table> tables = new ArrayList<Table>();
+ List<Table> tables = null;
startMultiTableFunction("get_multi_table", dbname, names);
try {
tables = executeWithRetry(new Command<List<Table>>() {
@@ -1242,7 +1277,7 @@ public class HiveMetaStore extends Thrif
} catch (Exception e) {
throw new MetaException(e.toString());
} finally {
- endFunction("get_multi_table");
+ endFunction("get_multi_table", tables != null);
}
return tables;
}
@@ -1251,7 +1286,7 @@ public class HiveMetaStore extends Thrif
public List<String> get_table_names_by_filter(
final String dbName, final String filter, final short maxTables)
throws MetaException, InvalidOperationException, UnknownDBException {
- List<String> tables = new ArrayList<String>();
+ List<String> tables = null;
startFunction("get_table_names_by_filter", ": db = " + dbName + ",
filter = " + filter);
try {
tables = executeWithRetry(new Command<List<String>>() {
@@ -1276,14 +1311,14 @@ public class HiveMetaStore extends Thrif
} catch (Exception e) {
throw new MetaException(e.toString());
} finally {
- endFunction("get_table_names_by_filter");
+ endFunction("get_table_names_by_filter", tables != null);
}
return tables;
}
public boolean set_table_parameters(String dbname, String name,
Map<String, String> params) throws NoSuchObjectException,
MetaException {
- endFunction(startTableFunction("set_table_parameters", dbname, name));
+ endFunction(startTableFunction("set_table_parameters", dbname, name),
false);
// TODO Auto-generated method stub
return false;
}
@@ -1384,7 +1419,7 @@ public class HiveMetaStore extends Thrif
assert(e instanceof RuntimeException);
throw (RuntimeException)e;
} finally {
- endFunction("append_partition");
+ endFunction("append_partition", ret != null);
}
return ret;
}
@@ -1445,7 +1480,7 @@ public class HiveMetaStore extends Thrif
assert(e instanceof RuntimeException);
throw (RuntimeException)e;
} finally {
- endFunction("add_partition");
+ endFunction("add_partition", ret != null);
}
return ret;
}
@@ -1584,7 +1619,7 @@ public class HiveMetaStore extends Thrif
assert(e instanceof RuntimeException);
throw (RuntimeException)e;
} finally {
- endFunction("add_partition");
+ endFunction("add_partition", ret != null);
}
return ret;
@@ -1661,7 +1696,7 @@ public class HiveMetaStore extends Thrif
startPartitionFunction("drop_partition", db_name, tbl_name, part_vals);
LOG.info("Partition values:" + part_vals);
- Boolean ret = null;
+ boolean ret = false;
try {
ret = executeWithRetry(new Command<Boolean>() {
@Override
@@ -1680,9 +1715,9 @@ public class HiveMetaStore extends Thrif
assert(e instanceof RuntimeException);
throw (RuntimeException)e;
} finally {
- endFunction("drop_partition");
+ endFunction("drop_partition", ret);
}
- return ret.booleanValue();
+ return ret;
}
@@ -1706,7 +1741,7 @@ public class HiveMetaStore extends Thrif
assert(e instanceof RuntimeException);
throw (RuntimeException)e;
} finally {
- endFunction("get_partition");
+ endFunction("get_partition", ret != null);
}
return ret;
}
@@ -1736,7 +1771,7 @@ public class HiveMetaStore extends Thrif
assert (e instanceof RuntimeException);
throw (RuntimeException) e;
} finally {
- endFunction("get_partition_with_auth");
+ endFunction("get_partition_with_auth", ret != null);
}
return ret;
}
@@ -1761,7 +1796,7 @@ public class HiveMetaStore extends Thrif
assert(e instanceof RuntimeException);
throw (RuntimeException)e;
} finally {
- endFunction("get_partitions");
+ endFunction("get_partitions", ret != null);
}
return ret;
@@ -1791,7 +1826,7 @@ public class HiveMetaStore extends Thrif
assert (e instanceof RuntimeException);
throw (RuntimeException) e;
} finally {
- endFunction("get_partitions_with_auth");
+ endFunction("get_partitions_with_auth", ret != null);
}
return ret;
@@ -1815,7 +1850,7 @@ public class HiveMetaStore extends Thrif
assert(e instanceof RuntimeException);
throw (RuntimeException)e;
} finally {
- endFunction("get_partition_names");
+ endFunction("get_partition_names", ret != null);
}
return ret;
}
@@ -1837,8 +1872,9 @@ public class HiveMetaStore extends Thrif
LOG.info("Old Partition values:" + part_vals);
}
+ boolean success = false;
try {
- executeWithRetry(new Command<Boolean>() {
+ success = executeWithRetry(new Command<Boolean>() {
@Override
public Boolean run(RawStore ms) throws Exception {
alter_partition_core(ms, db_name, tbl_name, part_vals, new_part);
@@ -1857,7 +1893,7 @@ public class HiveMetaStore extends Thrif
assert(e instanceof RuntimeException);
throw (RuntimeException)e;
} finally {
- endFunction("alter_partition");
+ endFunction("alter_partition", success);
}
return;
}
@@ -2007,7 +2043,7 @@ public class HiveMetaStore extends Thrif
public boolean create_index(Index index_def)
throws IndexAlreadyExistsException, MetaException {
- endFunction(startFunction("create_index"));
+ endFunction(startFunction("create_index"), false);
// TODO Auto-generated method stub
throw new MetaException("Not yet implemented");
}
@@ -2020,8 +2056,9 @@ public class HiveMetaStore extends Thrif
newIndex.putToParameters(Constants.DDL_TIME, Long.toString(System
.currentTimeMillis() / 1000));
+ boolean success = false;
try {
- executeWithRetry(new Command<Boolean>() {
+ success = executeWithRetry(new Command<Boolean>() {
@Override
public Boolean run(RawStore ms) throws Exception {
ms.alterIndex(dbname, base_table_name, index_name, newIndex);
@@ -2036,13 +2073,13 @@ public class HiveMetaStore extends Thrif
assert(e instanceof RuntimeException);
throw (RuntimeException)e;
} finally {
- endFunction("alter_index");
+ endFunction("alter_index", false);
}
return;
}
public String getVersion() throws TException {
- endFunction(startFunction("getVersion"));
+ endFunction(startFunction("getVersion"), true);
return "3.0";
}
@@ -2057,9 +2094,10 @@ public class HiveMetaStore extends Thrif
newTable.putToParameters(Constants.DDL_TIME, Long.toString(System
.currentTimeMillis() / 1000));
}
+ boolean success = false;
try {
Table oldt = get_table(dbname, name);
- boolean success = executeWithRetry(new Command<Boolean>() {
+ success = executeWithRetry(new Command<Boolean>() {
@Override
public Boolean run(RawStore ms) throws Exception {
alterHandler.alterTable(ms, wh, dbname, name, newTable);
@@ -2080,7 +2118,7 @@ public class HiveMetaStore extends Thrif
assert(e instanceof RuntimeException);
throw (RuntimeException)e;
} finally {
- endFunction("alter_table");
+ endFunction("alter_table", success);
}
}
@@ -2088,7 +2126,7 @@ public class HiveMetaStore extends Thrif
throws MetaException {
startFunction("get_tables", ": db=" + dbname + " pat=" + pattern);
- List<String> ret;
+ List<String> ret = null;
try {
ret = executeWithRetry(new Command<List<String>>() {
@Override
@@ -2102,7 +2140,7 @@ public class HiveMetaStore extends Thrif
assert(e instanceof RuntimeException);
throw (RuntimeException)e;
} finally {
- endFunction("get_tables");
+ endFunction("get_tables", ret != null);
}
return ret;
}
@@ -2110,7 +2148,7 @@ public class HiveMetaStore extends Thrif
public List<String> get_all_tables(final String dbname) throws
MetaException {
startFunction("get_all_tables", ": db=" + dbname);
- List<String> ret;
+ List<String> ret = null;
try {
ret = executeWithRetry(new Command<List<String>>() {
@Override
@@ -2124,7 +2162,7 @@ public class HiveMetaStore extends Thrif
assert(e instanceof RuntimeException);
throw (RuntimeException)e;
} finally {
- endFunction("get_all_tables");
+ endFunction("get_all_tables", ret != null);
}
return ret;
}
@@ -2136,6 +2174,7 @@ public class HiveMetaStore extends Thrif
String base_table_name = names[0];
Table tbl;
+ List<FieldSchema> ret = null;
try {
try {
tbl = get_table(db, base_table_name);
@@ -2145,19 +2184,21 @@ public class HiveMetaStore extends Thrif
boolean getColsFromSerDe = SerDeUtils.shouldGetColsFromSerDe(
tbl.getSd().getSerdeInfo().getSerializationLib());
if (!getColsFromSerDe) {
- return tbl.getSd().getCols();
+ ret = tbl.getSd().getCols();
} else {
try {
Deserializer s = MetaStoreUtils.getDeserializer(hiveConf, tbl);
- return MetaStoreUtils.getFieldsFromDeserializer(tableName, s);
+ ret = MetaStoreUtils.getFieldsFromDeserializer(tableName, s);
} catch (SerDeException e) {
StringUtils.stringifyException(e);
throw new MetaException(e.getMessage());
}
}
} finally {
- endFunction("get_fields");
+ endFunction("get_fields", ret != null);
}
+
+ return ret;
}
/**
@@ -2176,6 +2217,7 @@ public class HiveMetaStore extends Thrif
public List<FieldSchema> get_schema(String db, String tableName)
throws MetaException, UnknownTableException, UnknownDBException {
startFunction("get_schema", ": db=" + db + "tbl=" + tableName);
+ boolean success = false;
try {
String[] names = tableName.split("\\.");
String base_table_name = names[0];
@@ -2197,9 +2239,10 @@ public class HiveMetaStore extends Thrif
// whole schema
fieldSchemas.addAll(tbl.getPartitionKeys());
}
+ success = true;
return fieldSchemas;
} finally {
- endFunction("get_schema");
+ endFunction("get_schema", success);
}
}
@@ -2217,8 +2260,10 @@ public class HiveMetaStore extends Thrif
throws TException, ConfigValSecurityException {
startFunction("get_config_value", ": name=" + name + " defaultValue="
+ defaultValue);
+ boolean success = false;
try {
if (name == null) {
+ success = true;
return defaultValue;
}
// Allow only keys that start with hive.*, hdfs.*, mapred.* for
security
@@ -2236,9 +2281,10 @@ public class HiveMetaStore extends Thrif
+ "RuntimeException thrown in get_config_value - msg: "
+ e.getMessage() + " cause: " + e.getCause());
}
+ success = true;
return toReturn;
} finally {
- endFunction("get_config_value");
+ endFunction("get_config_value", success);
}
}
@@ -2310,7 +2356,7 @@ public class HiveMetaStore extends Thrif
assert(e instanceof RuntimeException);
throw (RuntimeException)e;
} finally {
- endFunction("get_partition_by_name");
+ endFunction("get_partition_by_name", ret != null);
}
return ret;
}
@@ -2342,7 +2388,7 @@ public class HiveMetaStore extends Thrif
assert(e instanceof RuntimeException);
throw (RuntimeException)e;
} finally {
- endFunction("append_partition_by_name");
+ endFunction("append_partition_by_name", ret != null);
}
return ret;
}
@@ -2369,7 +2415,7 @@ public class HiveMetaStore extends Thrif
startFunction("drop_partition_by_name", ": db=" + db_name + " tbl="
+ tbl_name + " part=" + part_name);
- Boolean ret = null;
+ boolean ret = false;
try {
ret = executeWithRetry(new Command<Boolean>() {
@Override
@@ -2388,10 +2434,10 @@ public class HiveMetaStore extends Thrif
assert(e instanceof RuntimeException);
throw (RuntimeException)e;
} finally {
- endFunction("drop_partition_by_name");
+ endFunction("drop_partition_by_name", ret);
}
- return ret.booleanValue();
+ return ret;
}
@Override
@@ -2399,13 +2445,17 @@ public class HiveMetaStore extends Thrif
final String tbl_name, final List<String> part_vals,
final short max_parts) throws MetaException, TException {
startPartitionFunction("get_partitions_ps", db_name, tbl_name,
part_vals);
+
+ List<Partition> ret = null;
try {
- return get_partitions_ps_with_auth(db_name, tbl_name, part_vals,
+ ret = get_partitions_ps_with_auth(db_name, tbl_name, part_vals,
max_parts, null, null);
}
finally {
- endFunction("get_partitions_ps");
+ endFunction("get_partitions_ps", ret != null);
}
+
+ return ret;
}
@Override
@@ -2415,7 +2465,7 @@ public class HiveMetaStore extends Thrif
final List<String> groupNames) throws MetaException, TException {
startPartitionFunction("get_partitions_ps_with_auth", db_name, tbl_name,
part_vals);
- List<Partition> ret;
+ List<Partition> ret = null;
try {
ret = executeWithRetry(new Command<List<Partition>>() {
@Override
@@ -2432,7 +2482,7 @@ public class HiveMetaStore extends Thrif
assert(e instanceof RuntimeException);
throw (RuntimeException)e;
} finally {
- endFunction("get_partitions_ps_with_auth");
+ endFunction("get_partitions_ps_with_auth", ret != null);
}
return ret;
}
@@ -2442,7 +2492,7 @@ public class HiveMetaStore extends Thrif
final String tbl_name, final List<String> part_vals, final short
max_parts)
throws MetaException, TException {
startPartitionFunction("get_partitions_names_ps", db_name, tbl_name,
part_vals);
- List<String> ret;
+ List<String> ret = null;
try {
ret = executeWithRetry(new Command<List<String>>() {
@Override
@@ -2456,7 +2506,7 @@ public class HiveMetaStore extends Thrif
assert(e instanceof RuntimeException);
throw (RuntimeException)e;
} finally {
- endFunction("get_partitions_names_ps");
+ endFunction("get_partitions_names_ps", ret != null);
}
return ret;
}
@@ -2505,7 +2555,7 @@ public class HiveMetaStore extends Thrif
assert(e instanceof RuntimeException);
throw (RuntimeException)e;
} finally {
- endFunction("add_index");
+ endFunction("add_index", ret != null);
}
return ret;
}
@@ -2574,7 +2624,7 @@ public class HiveMetaStore extends Thrif
startFunction("drop_index_by_name", ": db=" + dbName + " tbl="
+ tblName + " index=" + indexName);
- Boolean ret = null;
+ boolean ret = false;
try {
ret = executeWithRetry(new Command<Boolean>() {
@Override
@@ -2593,10 +2643,10 @@ public class HiveMetaStore extends Thrif
assert(e instanceof RuntimeException);
throw (RuntimeException)e;
} finally {
- endFunction("drop_index_by_name");
+ endFunction("drop_index_by_name", ret);
}
- return ret.booleanValue();
+ return ret;
}
private boolean drop_index_by_name_core(final RawStore ms,
@@ -2677,7 +2727,7 @@ public class HiveMetaStore extends Thrif
assert(e instanceof RuntimeException);
throw (RuntimeException)e;
} finally {
- endFunction("drop_index_by_name");
+ endFunction("drop_index_by_name", ret != null);
}
return ret;
}
@@ -2713,7 +2763,7 @@ public class HiveMetaStore extends Thrif
assert(e instanceof RuntimeException);
throw (RuntimeException)e;
} finally {
- endFunction("get_index_names");
+ endFunction("get_index_names", ret != null);
}
return ret;
}
@@ -2738,7 +2788,7 @@ public class HiveMetaStore extends Thrif
assert(e instanceof RuntimeException);
throw (RuntimeException)e;
} finally {
- endFunction("get_indexes");
+ endFunction("get_indexes", ret != null);
}
return ret;
}
@@ -2765,7 +2815,7 @@ public class HiveMetaStore extends Thrif
assert(e instanceof RuntimeException);
throw (RuntimeException)e;
} finally {
- endFunction("get_partitions_by_filter");
+ endFunction("get_partitions_by_filter", ret != null);
}
return ret;
}
@@ -2793,7 +2843,7 @@ public class HiveMetaStore extends Thrif
assert(e instanceof RuntimeException);
throw (RuntimeException)e;
} finally {
- endFunction("get_partitions_by_names");
+ endFunction("get_partitions_by_names", ret != null);
}
return ret;
}
@@ -3394,12 +3444,14 @@ public class HiveMetaStore extends Thrif
public void cancel_delegation_token(String token_str_form)
throws MetaException, TException {
startFunction("cancel_delegation_token");
+ boolean success = false;
try {
HiveMetaStore.cancelDelegationToken(token_str_form);
+ success = true;
} catch(IOException e) {
throw new MetaException(e.getMessage());
} finally {
- endFunction("cancel_delegation_token");
+ endFunction("cancel_delegation_token", success);
}
}
@@ -3407,13 +3459,15 @@ public class HiveMetaStore extends Thrif
public long renew_delegation_token(String token_str_form)
throws MetaException, TException {
startFunction("renew_delegation_token");
+ Long ret = null;
try {
- return HiveMetaStore.renewDelegationToken(token_str_form);
+ ret = HiveMetaStore.renewDelegationToken(token_str_form);
} catch(IOException e) {
throw new MetaException(e.getMessage());
} finally {
- endFunction("renew_delegation_token");
+ endFunction("renew_delegation_token", ret != null);
}
+ return ret;
}
@Override
@@ -3421,8 +3475,9 @@ public class HiveMetaStore extends Thrif
String renewer_kerberos_principal_name)
throws MetaException, TException {
startFunction("get_delegation_token");
+ String ret = null;
try {
- return
+ ret =
HiveMetaStore.getDelegationToken(token_owner,
renewer_kerberos_principal_name);
} catch(IOException e) {
@@ -3430,8 +3485,9 @@ public class HiveMetaStore extends Thrif
} catch (InterruptedException e) {
throw new MetaException(e.getMessage());
} finally {
- endFunction("get_delegation_token");
+ endFunction("get_delegation_token", ret != null);
}
+ return ret;
}
@Override
@@ -3440,9 +3496,9 @@ public class HiveMetaStore extends Thrif
MetaException,TException, NoSuchObjectException, UnknownDBException,
UnknownTableException,
InvalidPartitionException, UnknownPartitionException {
+ Table tbl = null;
try {
startPartitionFunction("markPartitionForEvent", db_name, tbl_name,
partName);
- Table tbl = null;
try{
tbl = executeWithRetry(new Command<Table>(){
@Override
@@ -3479,7 +3535,7 @@ public class HiveMetaStore extends Thrif
}
}
finally{
- endFunction("markPartitionForEvent");
+ endFunction("markPartitionForEvent", tbl != null);
}
}
@@ -3490,8 +3546,9 @@ public class HiveMetaStore extends Thrif
TException, UnknownPartitionException, InvalidPartitionException {
startPartitionFunction("isPartitionMarkedForEvent", db_name, tbl_name,
partName);
+ Boolean ret = null;
try {
- return executeWithRetry(new Command<Boolean>(){
+ ret = executeWithRetry(new Command<Boolean>(){
@Override
public Boolean run(RawStore ms) throws Exception {
return ms.isPartitionMarkedForEvent(db_name, tbl_name, partName,
evtType);
@@ -3519,8 +3576,10 @@ public class HiveMetaStore extends Thrif
}
}
finally{
- endFunction("isPartitionMarkedForEvent");
+ endFunction("isPartitionMarkedForEvent", ret != null);
}
+
+ return ret;
}
}
Added:
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionContext.java
URL:
http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionContext.java?rev=1209368&view=auto
==============================================================================
---
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionContext.java
(added)
+++
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionContext.java
Fri Dec 2 05:55:26 2011
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+/**
+ * Base class which provides context to implementations of
MetaStoreEndFunctionListener
+ */
+
+public class MetaStoreEndFunctionContext {
+
+ /**
+ * whether method was successful or not.
+ */
+ private final boolean success;
+
+ public MetaStoreEndFunctionContext(boolean success) {
+ this.success = success;
+ }
+
+ /**
+ * @return whether or not the method succeeded.
+ */
+ public boolean isSuccess() {
+ return success;
+ }
+
+}
Added:
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionListener.java
URL:
http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionListener.java?rev=1209368&view=auto
==============================================================================
---
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionListener.java
(added)
+++
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionListener.java
Fri Dec 2 05:55:26 2011
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import java.util.AbstractMap;
+
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * This abstract class needs to be extended to provide implementation of
actions that need
+ * to be performed when a function ends. These methods are called whenever a
function ends.
+ *
+ * It also provides a way to add fb303 counters through the exportCounters
method.
+ */
+
+public abstract class MetaStoreEndFunctionListener implements Configurable {
+
+ private Configuration conf;
+
+ public MetaStoreEndFunctionListener(Configuration config){
+ this.conf = config;
+ }
+
+ public abstract void onEndFunction(String functionName,
MetaStoreEndFunctionContext context);
+
+ // Unless this is overridden, it does nothing
+ public void exportCounters(AbstractMap<String, Long> counters) {
+ }
+
+ @Override
+ public Configuration getConf() {
+ return this.conf;
+ }
+
+ @Override
+ public void setConf(Configuration config) {
+ this.conf = config;
+ }
+
+
+}
Modified:
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
URL:
http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java?rev=1209368&r1=1209367&r2=1209368&view=diff
==============================================================================
---
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
(original)
+++
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
Fri Dec 2 05:55:26 2011
@@ -960,27 +960,28 @@ public class MetaStoreUtils {
/**
* create listener instances as per the configuration.
+ *
+ * @param clazz
* @param conf
+ * @param listenerImplList
* @return
* @throws MetaException
*/
- static List<MetaStoreEventListener> getMetaStoreListener (HiveConf conf)
- throws MetaException {
+ static <T> List<T> getMetaStoreListeners(Class<T> clazz,
+ HiveConf conf, String listenerImplList) throws MetaException {
- List<MetaStoreEventListener> listeners = new
ArrayList<MetaStoreEventListener>();
- String listenerImplList =
conf.getVar(HiveConf.ConfVars.METASTORE_EVENT_LISTENERS);
+ List<T> listeners = new ArrayList<T>();
listenerImplList = listenerImplList.trim();
if (listenerImplList.equals("")) {
return listeners;
-}
+ }
String[] listenerImpls = listenerImplList.split(",");
for (String listenerImpl : listenerImpls) {
try {
- MetaStoreEventListener listener = (MetaStoreEventListener)
Class.forName(
+ T listener = (T) Class.forName(
listenerImpl.trim(), true,
JavaUtils.getClassLoader()).getConstructor(
Configuration.class).newInstance(conf);
- listener.setConf(conf);
listeners.add(listener);
} catch (Exception e) {
throw new MetaException("Failed to instantiate listener named: "+
