[
https://issues.apache.org/jira/browse/DRILL-4726?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15469088#comment-15469088
]
ASF GitHub Bot commented on DRILL-4726:
---------------------------------------
Github user paul-rogers commented on a diff in the pull request:
https://github.com/apache/drill/pull/574#discussion_r77743650
--- Diff:
exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
---
@@ -186,4 +226,105 @@ public boolean isFunctionComplexOutput(String name) {
return false;
}
+ public RemoteFunctionRegistry getRemoteFunctionRegistry() {
+ return remoteFunctionRegistry;
+ }
+
+ public List<Func> validate(Path path) throws IOException {
+ URL url = path.toUri().toURL();
+ URL[] urls = {url};
+ ClassLoader classLoader = new URLClassLoader(urls);
+ return drillFuncRegistry.validate(path.getName(), scan(classLoader,
path, urls));
+ }
+
+ public void register(String jarName, ScanResult classpathScan,
ClassLoader classLoader) {
+ drillFuncRegistry.register(jarName, classpathScan, classLoader);
+ }
+
+ public void unregister(String jarName) {
+ drillFuncRegistry.unregister(jarName);
+ }
+
+ /**
+ * Loads all missing functions from remote registry.
+ * Compares list of already registered jars and remote jars, loads
missing jars.
+ * Missing jars are stores in local DRILL_UDF_DIR.
+ *
+ * @return true if at least functions from one jar were loaded
+ */
+ public boolean loadRemoteFunctions() {
+ List<String> missingJars = Lists.newArrayList();
+ Registry registry = remoteFunctionRegistry.getRegistry();
+
+ List<String> localJars = drillFuncRegistry.getAllJarNames();
+ for (Jar jar : registry.getJarList()) {
+ if (!localJars.contains(jar.getName())) {
+ missingJars.add(jar.getName());
+ }
+ }
+
+ for (String jarName : missingJars) {
+ try {
+ Path localUdfArea = new Path(new File(getUdfDir()).toURI());
+ Path registryArea = remoteFunctionRegistry.getRegistryArea();
+ FileSystem fs = remoteFunctionRegistry.getFs();
+
+ String sourceName = JarUtil.getSourceName(jarName);
+
+ Path remoteBinary = new Path(registryArea, jarName);
+ Path remoteSource = new Path(registryArea, sourceName);
+
+ Path binary = new Path(localUdfArea, jarName);
+ Path source = new Path(localUdfArea, sourceName);
+
+ fs.copyToLocalFile(remoteBinary, binary);
+ fs.copyToLocalFile(remoteSource, source);
+
+ URL[] urls = {binary.toUri().toURL(), source.toUri().toURL()};
+ ClassLoader classLoader = new URLClassLoader(urls);
+ ScanResult scanResult = scan(classLoader, binary, urls);
+ drillFuncRegistry.register(jarName, scanResult, classLoader);
+ } catch (IOException | FunctionValidationException e) {
+ logger.error("Problem during remote functions load from {}",
jarName, e);
+ }
+ }
+
+ return missingJars.size() > 0;
+ }
+
+ private ScanResult scan(ClassLoader classLoader, Path path, URL[] urls)
throws IOException {
+ Enumeration<URL> e =
classLoader.getResources(CommonConstants.DRILL_JAR_MARKER_FILE_RESOURCE_PATHNAME);
+ while (e.hasMoreElements()) {
+ URL res = e.nextElement();
+ if (res.getPath().contains(path.toUri().getPath())) {
+ DrillConfig drillConfig =
DrillConfig.create(ConfigFactory.parseURL(res));
+ return RunTimeScan.dynamicPackageScan(drillConfig,
Sets.newHashSet(urls));
+ }
+ }
+ throw new FunctionValidationException(String.format("Marker file %s is
missing in %s.",
+ CommonConstants.DRILL_JAR_MARKER_FILE_RESOURCE_PATHNAME,
path.getName()));
+ }
+
+ private static String getUdfDir() {
+ return Preconditions.checkNotNull(System.getenv("DRILL_UDF_DIR"),
"DRILL_UDF_DIR variable is not set");
+ }
+
+ /**
+ * Fires when jar name is submitted for unregistration.
+ * Will unregister all functions associated with the jar name
+ * and delete binary and source associated with the jar from local
DRILL_UDF_DIR.
+ */
+ public class UnregistrationListener implements TransientStoreListener {
+
+ @Override
+ public void onChange(TransientStoreEvent event) {
+ String jarName = (String) event.getValue();
+ String sourceName = JarUtil.getSourceName(jarName);
+ String localDir = getUdfDir();
+ FileUtils.deleteQuietly(new File(localDir, jarName));
+ FileUtils.deleteQuietly(new File(localDir, sourceName));
+ drillFuncRegistry.unregister(jarName);
--- End diff --
Race condition. We delete the jars before we unregister. Another thread can
check the registry, then look for the jars, and not find them. Is
synchronization needed here?
> Dynamic UDFs support
> --------------------
>
> Key: DRILL-4726
> URL: https://issues.apache.org/jira/browse/DRILL-4726
> Project: Apache Drill
> Issue Type: New Feature
> Affects Versions: 1.6.0
> Reporter: Arina Ielchiieva
> Assignee: Arina Ielchiieva
> Fix For: Future
>
>
> Allow register UDFs without restart of Drillbits.
> Design is described in document below:
> https://docs.google.com/document/d/1FfyJtWae5TLuyheHCfldYUpCdeIezR2RlNsrOTYyAB4/edit?usp=sharing
>
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
