This is an automated email from the ASF dual-hosted git repository.
mustafaiman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 25892ea HIVE-25040: Drop database cascade cannot remove persistent
functions (Mustafa Iman, reviewed by Panos Garefalakis)
25892ea is described below
commit 25892ea409b3351acbd409d1f66f399a43614798
Author: Mustafa Iman <mustafai...@gmail.com>
AuthorDate: Tue Apr 20 17:47:00 2021 -0700
HIVE-25040: Drop database cascade cannot remove persistent functions
(Mustafa Iman, reviewed by Panos Garefalakis)
---
.../session/TestUdfClassLoaderAcrossSessions.java | 92 +++++++++++++++++++++
itests/hive-unit/testUdf/DummyUDF.class | Bin 0 -> 532 bytes
itests/hive-unit/testUdf/DummyUDF.jar | Bin 0 -> 837 bytes
itests/hive-unit/testUdf/DummyUDF.java | 35 ++++++++
.../org/apache/hadoop/hive/ql/exec/Registry.java | 29 ++++---
5 files changed, 143 insertions(+), 13 deletions(-)
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/session/TestUdfClassLoaderAcrossSessions.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/session/TestUdfClassLoaderAcrossSessions.java
new file mode 100644
index 0000000..f619e6f
--- /dev/null
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/session/TestUdfClassLoaderAcrossSessions.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.session;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.DriverFactory;
+import org.apache.hadoop.hive.ql.IDriver;
+import org.apache.hadoop.hive.ql.processors.CommandProcessorException;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class TestUdfClassLoaderAcrossSessions {
+
+ @Test
+ public void testDropDatabaseCascadeDoesNotThrow() throws
CommandProcessorException, IOException {
+ HiveConf conf = new HiveConf(this.getClass());
+ conf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
+ SessionState.start(conf);
+ IDriver driver = DriverFactory.newDriver(conf);
+
+ driver.run("CREATE DATABASE udfacross1");
+ driver.run("USE udfacross1");
+ driver.run("CREATE FUNCTION dummyFunc AS 'DummyUDF' USING JAR
'testUdf/DummyUDF.jar'");
+ SessionState.get().close();
+ SessionState.start(conf);
+ driver.run("DROP DATABASE udfacross1 CASCADE");
+ assertFunctionRemoved(driver, "udfacross1.dummyFunc");
+ }
+
+ @Test
+ public void testDropFunctionDoesNotThrow() throws CommandProcessorException,
IOException {
+ HiveConf conf = new HiveConf(this.getClass());
+ conf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
+ SessionState.start(conf);
+ IDriver driver = DriverFactory.newDriver(conf);
+
+ driver.run("CREATE DATABASE udfacross2");
+ driver.run("USE udfacross2");
+ driver.run("CREATE FUNCTION dummyFunc AS 'DummyUDF' USING JAR
'testUdf/DummyUDF.jar'");
+ SessionState.get().close();
+ SessionState.start(conf);
+ driver.run("DROP FUNCTION udfacross2.dummyFunc");
+ driver.run("DROP DATABASE udfacross2");
+ assertFunctionRemoved(driver, "udfacross2.dummyFunc");
+ }
+
+ @Test
+ public void testUseBeforeDropDatabaseCascadeDoesNotThrow() throws
CommandProcessorException, IOException {
+ HiveConf conf = new HiveConf(this.getClass());
+ conf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
+ SessionState.start(conf);
+ IDriver driver = DriverFactory.newDriver(conf);
+
+ driver.run("CREATE DATABASE udfacross3");
+ driver.run("USE udfacross3");
+ driver.run("CREATE FUNCTION dummyFunc AS 'DummyUDF' USING JAR
'testUdf/DummyUDF.jar'");
+ SessionState.get().close();
+ SessionState.start(conf);
+ driver.run("SELECT udfacross3.dummyFunc(true)");
+ driver.run("DROP DATABASE udfacross3 CASCADE");
+ assertFunctionRemoved(driver, "udfacross3.dummyFunc");
+ }
+
+ private void assertFunctionRemoved(IDriver driver, String name) throws
CommandProcessorException, IOException {
+ driver.run("DESCRIBE FUNCTION " + name);
+ List<String> result = new ArrayList<>();
+ driver.getResults(result);
+ if (result.size() > 1) {
+ Assert.fail("DESCRIBE FUNCTION " + name + ":\n" +
result.toArray().toString());
+ }
+ Assert.assertEquals("Function '" + name +"' does not exist."
,result.get(0));
+ }
+}
diff --git a/itests/hive-unit/testUdf/DummyUDF.class
b/itests/hive-unit/testUdf/DummyUDF.class
new file mode 100644
index 0000000..4dfa4ac
Binary files /dev/null and b/itests/hive-unit/testUdf/DummyUDF.class differ
diff --git a/itests/hive-unit/testUdf/DummyUDF.jar
b/itests/hive-unit/testUdf/DummyUDF.jar
new file mode 100644
index 0000000..f71235d
Binary files /dev/null and b/itests/hive-unit/testUdf/DummyUDF.jar differ
diff --git a/itests/hive-unit/testUdf/DummyUDF.java
b/itests/hive-unit/testUdf/DummyUDF.java
new file mode 100644
index 0000000..0999d45
--- /dev/null
+++ b/itests/hive-unit/testUdf/DummyUDF.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.UDF;
+
+/**
+ * This UDF is packaged in a standalone jar to be used in {@link
org.apache.hadoop.hive.ql.session.TestUdfClassLoaderAcrossSessions}
+ * After changing this file, recompile and rejar it manually into DummyUDF.jar
+ * Run:
+ * - javac -cp <hiveroot>/ql/target/hive-exec-<version>.jar DummyUDF.java
+ * - jar cf DummyUDF.jar DummyUDF.class
+ */
+@Description(name = "dummy",
+ value = "_FUNC_(col) this UDF does nothing")
+public class DummyUDF extends UDF {
+ public int evaluate(Boolean b) {
+ return b ? 1 : 0;
+ }
+}
\ No newline at end of file
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java
index e72356c..d70cccd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java
@@ -553,21 +553,19 @@ public class Registry {
Integer refCount = persistent.get(functionClass);
persistent.put(functionClass, Integer.valueOf(refCount == null ? 1 :
refCount + 1));
}
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException(e);
} finally {
lock.unlock();
}
}
- private Class<?> getPermanentUdfClass(FunctionInfo function) {
+ private Class<?> getPermanentUdfClass(FunctionInfo function) throws
ClassNotFoundException {
Class<?> functionClass = function.getFunctionClass();
if (functionClass == null) {
// Expected for permanent UDFs at this point.
ClassLoader loader = Utilities.getSessionSpecifiedClassLoader();
- try {
- functionClass = Class.forName(function.getClassName(), true, loader);
- } catch (ClassNotFoundException ex) {
- throw new RuntimeException(ex);
- }
+ functionClass = Class.forName(function.getClassName(), true, loader);
}
return functionClass;
}
@@ -593,14 +591,19 @@ public class Registry {
}
private void removePersistentFunctionUnderLock(FunctionInfo fi) {
- Class<?> functionClass = getPermanentUdfClass(fi);
- Integer refCount = persistent.get(functionClass);
- if (refCount != null) {
- if (refCount == 1) {
- persistent.remove(functionClass);
- } else {
- persistent.put(functionClass, Integer.valueOf(refCount - 1));
+ try {
+ Class<?> functionClass = getPermanentUdfClass(fi);
+ Integer refCount = persistent.get(functionClass);
+ if (refCount != null) {
+ if (refCount == 1) {
+ persistent.remove(functionClass);
+ } else {
+ persistent.put(functionClass, Integer.valueOf(refCount - 1));
+ }
}
+ } catch (ClassNotFoundException e) {
+ LOG.debug("Associated class could not be found when dropping a custom
UDF {}." +
+ "This may happen if this UDF was never used in this session.",
fi.getDisplayName());
}
}
