Author: hashutosh
Date: Wed Feb 19 23:50:32 2014
New Revision: 1569985
URL: http://svn.apache.org/r1569985
Log:
HIVE-6330 : Metastore support for permanent UDFs (Jason Dere via Ashutosh
Chauhan)
Added:
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Function.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/FunctionType.java
hive/trunk/metastore/src/model/org/apache/hadoop/hive/metastore/model/MFunction.java
hive/trunk/ql/src/test/queries/clientnegative/create_function_nonexistent_class.q
hive/trunk/ql/src/test/queries/clientnegative/create_function_nonexistent_db.q
hive/trunk/ql/src/test/queries/clientnegative/create_function_nonudf_class.q
hive/trunk/ql/src/test/queries/clientnegative/drop_func_nonexistent.q
hive/trunk/ql/src/test/queries/clientpositive/create_func1.q
hive/trunk/ql/src/test/results/clientnegative/create_function_nonexistent_class.q.out
hive/trunk/ql/src/test/results/clientnegative/create_function_nonexistent_db.q.out
hive/trunk/ql/src/test/results/clientnegative/create_function_nonudf_class.q.out
hive/trunk/ql/src/test/results/clientnegative/drop_func_nonexistent.q.out
hive/trunk/ql/src/test/results/clientpositive/create_func1.q.out
Modified:
hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
hive/trunk/metastore/if/hive_metastore.thrift
hive/trunk/metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.cpp
hive/trunk/metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.h
hive/trunk/metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore_server.skeleton.cpp
hive/trunk/metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
hive/trunk/metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/AddPartitionsRequest.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/AddPartitionsResult.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ColumnStatistics.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/DropPartitionsResult.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/HiveObjectRef.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Partition.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/PartitionsByExprResult.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/PartitionsStatsRequest.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/PartitionsStatsResult.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/PrincipalPrivilegeSet.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/PrivilegeBag.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/RequestPartsSpec.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Schema.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/SkewedInfo.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/StorageDescriptor.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Table.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/TableStatsRequest.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/TableStatsResult.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Type.java
hive/trunk/metastore/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php
hive/trunk/metastore/src/gen/thrift/gen-php/metastore/Types.php
hive/trunk/metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore-remote
hive/trunk/metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore.py
hive/trunk/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py
hive/trunk/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
hive/trunk/metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java
hive/trunk/metastore/src/model/package.jdo
hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionUtils.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/FunctionSemanticAnalyzer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/MacroSemanticAnalyzer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateFunctionDesc.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/DropFunctionDesc.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFunctionRegistry.java
hive/trunk/ql/src/test/results/clientnegative/database_drop_not_empty.q.out
hive/trunk/ql/src/test/results/clientnegative/database_drop_not_empty_restrict.q.out
hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out
Modified:
hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
URL:
http://svn.apache.org/viewvc/hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java?rev=1569985&r1=1569984&r2=1569985&view=diff
==============================================================================
---
hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
(original)
+++
hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
Wed Feb 19 23:50:32 2014
@@ -48,6 +48,7 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.Function;
import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
import org.apache.hadoop.hive.metastore.api.MetaException;
@@ -62,6 +63,7 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.api.Type;
import org.apache.hadoop.hive.metastore.api.UnknownDBException;
+import org.apache.hadoop.hive.metastore.api.FunctionType;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.serde.serdeConstants;
@@ -2489,6 +2491,66 @@ public abstract class TestHiveMetaStore
}
}
+ public void testSimpleFunction() throws Exception {
+ String dbName = "test_db";
+ String funcName = "test_func";
+ String className = "org.apache.hadoop.hive.ql.udf.generic.GenericUDFUpper";
+ String owner = "test_owner";
+ PrincipalType ownerType = PrincipalType.USER;
+ int createTime = (int) (System.currentTimeMillis() / 1000);
+ FunctionType funcType = FunctionType.JAVA;
+
+ try {
+ cleanUp(dbName, null, null);
+
+ createDb(dbName);
+
+ createFunction(dbName, funcName, className, owner, ownerType,
createTime, funcType);
+
+ // Try the different getters
+
+ // getFunction()
+ Function func = client.getFunction(dbName, funcName);
+ assertEquals("function db name", dbName, func.getDbName());
+ assertEquals("function name", funcName, func.getFunctionName());
+ assertEquals("function class name", className, func.getClassName());
+ assertEquals("function owner name", owner, func.getOwnerName());
+ assertEquals("function owner type", PrincipalType.USER,
func.getOwnerType());
+ assertEquals("function type", funcType, func.getFunctionType());
+
+ boolean gotException = false;
+ try {
+ func = client.getFunction(dbName, "nonexistent_func");
+ } catch (MetaException e) {
+ // expected failure
+ gotException = true;
+ }
+ assertEquals(true, gotException);
+
+ // getFunctions()
+ List<String> funcs = client.getFunctions(dbName, "*_func");
+ assertEquals(1, funcs.size());
+ assertEquals(funcName, funcs.get(0));
+
+ funcs = client.getFunctions(dbName, "nonexistent_func");
+ assertEquals(0, funcs.size());
+
+ // dropFunction()
+ client.dropFunction(dbName, funcName);
+
+ // Confirm that the function is now gone
+ funcs = client.getFunctions(dbName, funcName);
+ assertEquals(0, funcs.size());
+
+ } catch (Exception e) {
+ System.err.println(StringUtils.stringifyException(e));
+ System.err.println("testConcurrentMetastores() failed.");
+ throw e;
+ } finally {
+ silentDropDatabase(dbName);
+ }
+ }
+
/**
@@ -2672,4 +2734,12 @@ public abstract class TestHiveMetaStore
assertEquals(db.getOwnerName(), HiveMetaStore.PUBLIC);
assertEquals(db.getOwnerType(), PrincipalType.ROLE);
}
+
+ private void createFunction(String dbName, String funcName, String className,
+ String ownerName, PrincipalType ownerType, int createTime,
+ org.apache.hadoop.hive.metastore.api.FunctionType functionType) throws
Exception {
+ Function func = new Function(funcName, dbName, className,
+ ownerName, ownerType, createTime, functionType);
+ client.createFunction(func);
+ }
}
Modified:
hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
URL:
http://svn.apache.org/viewvc/hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java?rev=1569985&r1=1569984&r2=1569985&view=diff
==============================================================================
---
hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
(original)
+++
hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
Wed Feb 19 23:50:32 2014
@@ -582,7 +582,8 @@ public class QTestUtil {
}
}
if (!DEFAULT_DATABASE_NAME.equals(dbName)) {
- db.dropDatabase(dbName);
+ // Drop cascade, may need to drop functions
+ db.dropDatabase(dbName, true, true, true);
}
}
SessionState.get().setCurrentDatabase(DEFAULT_DATABASE_NAME);
@@ -650,6 +651,17 @@ public class QTestUtil {
return;
}
+ private void runCmd(String cmd) throws Exception {
+ int ecode = 0;
+ ecode = drv.run(cmd).getResponseCode();
+ drv.close();
+ if (ecode != 0) {
+ throw new Exception("command: " + cmd
+ + " failed with exit code= " + ecode);
+ }
+ return;
+ }
+
public void createSources() throws Exception {
startSessionState();
@@ -758,6 +770,10 @@ public class QTestUtil {
runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toUri().getPath()
+ "' INTO TABLE "+AllVectorTypesRecord.TABLE_NAME);
+ runCmd("DROP FUNCTION IF EXISTS qtest_get_java_boolean ");
+ runCmd("CREATE FUNCTION qtest_get_java_boolean "
+ + " AS
'org.apache.hadoop.hive.ql.udf.generic.GenericUDFTestGetJavaBoolean'");
+
conf.setBoolean("hive.test.init.phase", false);
}
Modified: hive/trunk/metastore/if/hive_metastore.thrift
URL:
http://svn.apache.org/viewvc/hive/trunk/metastore/if/hive_metastore.thrift?rev=1569985&r1=1569984&r2=1569985&view=diff
==============================================================================
--- hive/trunk/metastore/if/hive_metastore.thrift (original)
+++ hive/trunk/metastore/if/hive_metastore.thrift Wed Feb 19 23:50:32 2014
@@ -356,6 +356,21 @@ struct DropPartitionsRequest {
8: optional bool needResult=true
}
+enum FunctionType {
+ JAVA = 1,
+}
+
+// User-defined function
+struct Function {
+ 1: string functionName,
+ 2: string dbName,
+ 3: string className,
+ 4: string ownerName,
+ 5: PrincipalType ownerType,
+ 6: i32 createTime,
+ 7: FunctionType functionType,
+}
+
exception MetaException {
1: string message
}
@@ -693,6 +708,27 @@ service ThriftHiveMetastore extends fb30
(1:NoSuchObjectException o1, 2:MetaException o2,
3:InvalidObjectException o3,
4:InvalidInputException o4)
+ //
+ // user-defined functions
+ //
+
+ void create_function(1:Function func)
+ throws (1:AlreadyExistsException o1,
+ 2:InvalidObjectException o2,
+ 3:MetaException o3,
+ 4:NoSuchObjectException o4)
+
+ void drop_function(1:string dbName, 2:string funcName)
+ throws (1:NoSuchObjectException o1, 2:MetaException o3)
+
+ void alter_function(1:string dbName, 2:string funcName, 3:Function newFunc)
+ throws (1:InvalidOperationException o1, 2:MetaException o2)
+
+ list<string> get_functions(1:string dbName, 2:string pattern)
+ throws (1:MetaException o1)
+ Function get_function(1:string dbName, 2:string funcName)
+ throws (1:MetaException o1, 2:NoSuchObjectException o2)
+
//authorization privileges
bool create_role(1:Role role) throws(1:MetaException o1)
