Author: kturner
Date: Mon Jan 7 23:51:06 2013
New Revision: 1430092
URL: http://svn.apache.org/viewvc?rev=1430092&view=rev
Log:
ACCUMULO-869 added post delegation for per table classloaders
Added:
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/PostDelegatingVFSClassLoader.java
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/master/thrift/TabletServerStatus.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/ContextManager.java
accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoaderTest.java
accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ContextManagerTest.java
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java?rev=1430092&r1=1430091&r2=1430092&view=diff
==============================================================================
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java
Mon Jan 7 23:51:06 2013
@@ -299,7 +299,12 @@ public enum Property {
//VFS ClassLoader properties
VFS_CLASSLOADER_SYSTEM_CLASSPATH_PROPERTY(AccumuloVFSClassLoader.VFS_CLASSLOADER_SYSTEM_CLASSPATH_PROPERTY,
"", PropertyType.STRING,
"Configuration for a system level vfs classloader. Accumulo jar can be
configured here and loaded out of HDFS."),
-
VFS_CONTEXT_CLASSPATH_PROPERTY(AccumuloVFSClassLoader.VFS_CONTEXT_CLASSPATH_PROPERTY,
null, PropertyType.PREFIX, "Classpath for this context");
+
VFS_CONTEXT_CLASSPATH_PROPERTY(AccumuloVFSClassLoader.VFS_CONTEXT_CLASSPATH_PROPERTY,
null, PropertyType.PREFIX,
+ "Properties in this category are define a classpath. These properties
start with the category prefix, followed by a context name. "
+ + "The value is a comma seperated list of URIs. Supports full regex
on filename alone. For example
general.vfs.context.classpath.cx1=hdfs://nn1:9902/mylibdir/*.jar. "
+ + "You can enable post delegation for a context, which will load
classes from the context first instead of the parent first. "
+ + "Do this by setting
general.vfs.context.classpath.<name>.delegation=post, where <name> is your
context name. "
+ + "If delegation is not specified, it defaults to loading from
parent classloader first.");
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/master/thrift/TabletServerStatus.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/master/thrift/TabletServerStatus.java?rev=1430092&r1=1430091&r2=1430092&view=diff
==============================================================================
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/master/thrift/TabletServerStatus.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/master/thrift/TabletServerStatus.java
Mon Jan 7 23:51:06 2013
@@ -1127,7 +1127,7 @@ import org.slf4j.LoggerFactory;
for (int _i1 = 0; _i1 < _map0.size; ++_i1)
{
String _key2; // required
- TableInfo _val3; // optional
+ TableInfo _val3; // required
_key2 = iprot.readString();
_val3 = new TableInfo();
_val3.read(iprot);
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java?rev=1430092&r1=1430091&r2=1430092&view=diff
==============================================================================
---
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
(original)
+++
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
Mon Jan 7 23:51:06 2013
@@ -200,6 +200,7 @@ import org.apache.accumulo.server.zookee
import org.apache.accumulo.start.Platform;
import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
import org.apache.accumulo.start.classloader.vfs.ContextManager;
+import org.apache.accumulo.start.classloader.vfs.ContextManager.ContextConfig;
import org.apache.commons.collections.map.LRUMap;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
@@ -3059,25 +3060,29 @@ public class TabletServer extends Abstra
}
try {
- AccumuloVFSClassLoader.getContextManager().setContextConfig(new
ContextManager.ContextConfig() {
-
- @Override
- public boolean isIsolated(String context) {
- return false;
- }
-
+ AccumuloVFSClassLoader.getContextManager().setContextConfig(new
ContextManager.ContextsConfig() {
@Override
- public String getContextURIs(String context) {
+ public ContextConfig getContextConfig(String context) {
String key = Property.VFS_CONTEXT_CLASSPATH_PROPERTY.getKey() +
context;
+ String uris = null;
+ boolean preDelegate = true;
+
Iterator<Entry<String,String>> iter =
getSystemConfiguration().iterator();
while (iter.hasNext()) {
Entry<String,String> entry = iter.next();
if (entry.getKey().equals(key)) {
- return entry.getValue();
+ uris = entry.getValue();
+ }
+
+ if (entry.getKey().equals(key + ".delegation") &&
entry.getValue().trim().equalsIgnoreCase("post")) {
+ preDelegate = false;
}
}
+ if (uris != null)
+ return new ContextConfig(uris, preDelegate);
+
return null;
}
});
Modified:
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java?rev=1430092&r1=1430091&r2=1430092&view=diff
==============================================================================
---
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java
(original)
+++
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java
Mon Jan 7 23:51:06 2013
@@ -46,13 +46,19 @@ public class AccumuloReloadingVFSClassLo
private ReloadingClassLoader parent = null;
private DefaultFileMonitor monitor = null;
private VFSClassLoader cl = null;
+ private boolean preDelegate;
@Override
public synchronized ClassLoader getClassLoader() {
if (cl == null || cl.getParent() != parent.getClassLoader()) {
try {
files = AccumuloVFSClassLoader.resolve(vfs, uris);
- setClassloader(new VFSClassLoader(files, vfs,
parent.getClassLoader()));
+
+ if (preDelegate)
+ cl = new VFSClassLoader(files, vfs, parent.getClassLoader());
+ else
+ cl = new PostDelegatingVFSClassLoader(files, vfs,
parent.getClassLoader());
+
} catch (FileSystemException fse) {
throw new RuntimeException(fse);
}
@@ -66,16 +72,21 @@ public class AccumuloReloadingVFSClassLo
}
- public AccumuloReloadingVFSClassLoader(String uris, FileSystemManager vfs,
ReloadingClassLoader parent, long monitorDelay) throws FileSystemException {
+ public AccumuloReloadingVFSClassLoader(String uris, FileSystemManager vfs,
ReloadingClassLoader parent, long monitorDelay, boolean preDelegate)
+ throws FileSystemException {
this.uris = uris;
this.vfs = vfs;
this.parent = parent;
+ this.preDelegate = preDelegate;
ArrayList<FileObject> pathsToMonitor = new ArrayList<FileObject>();
files = AccumuloVFSClassLoader.resolve(vfs, uris, pathsToMonitor);
- cl = new VFSClassLoader(files, vfs, parent.getClassLoader());
+ if (preDelegate)
+ cl = new VFSClassLoader(files, vfs, parent.getClassLoader());
+ else
+ cl = new PostDelegatingVFSClassLoader(files, vfs,
parent.getClassLoader());
monitor = new DefaultFileMonitor(this);
monitor.setDelay(monitorDelay);
@@ -87,8 +98,9 @@ public class AccumuloReloadingVFSClassLo
monitor.start();
}
- public AccumuloReloadingVFSClassLoader(String uris, FileSystemManager vfs,
final ReloadingClassLoader parent) throws FileSystemException {
- this(uris, vfs, parent, DEFAULT_TIMEOUT);
+ public AccumuloReloadingVFSClassLoader(String uris, FileSystemManager vfs,
final ReloadingClassLoader parent, boolean preDelegate)
+ throws FileSystemException {
+ this(uris, vfs, parent, DEFAULT_TIMEOUT, preDelegate);
}
public FileObject[] getFiles() {
Modified:
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java?rev=1430092&r1=1430091&r2=1430092&view=diff
==============================================================================
---
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
(original)
+++
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
Mon Jan 7 23:51:06 2013
@@ -175,7 +175,7 @@ public class AccumuloVFSClassLoader {
return wrapper;
// TODO monitor time for lib/ext was 1 sec... should this be configurable?
- return new AccumuloReloadingVFSClassLoader(dynamicCPath, vfs, wrapper,
1000);
+ return new AccumuloReloadingVFSClassLoader(dynamicCPath, vfs, wrapper,
1000, true);
}
public static ClassLoader getClassLoader() throws IOException {
Modified:
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/ContextManager.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/ContextManager.java?rev=1430092&r1=1430091&r2=1430092&view=diff
==============================================================================
---
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/ContextManager.java
(original)
+++
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/ContextManager.java
Mon Jan 7 23:51:06 2013
@@ -29,22 +29,22 @@ public class ContextManager {
// there is a lock per context so that one context can initialize w/o
blocking another context
private class Context {
AccumuloReloadingVFSClassLoader loader;
- String uris;
+ ContextConfig cconfig;
boolean closed = false;
- Context(String uris) {
- this.uris = uris;
+ Context(ContextConfig cconfig) {
+ this.cconfig = cconfig;
}
- synchronized AccumuloReloadingVFSClassLoader getClassLoader() throws
FileSystemException {
+ synchronized ClassLoader getClassLoader() throws FileSystemException {
if (closed)
return null;
if (loader == null) {
- loader = new AccumuloReloadingVFSClassLoader(uris, vfs, parent);
+ loader = new AccumuloReloadingVFSClassLoader(cconfig.uris, vfs,
parent, cconfig.preDelegation);
}
- return loader;
+ return loader.getClassLoader();
}
synchronized void close() {
@@ -56,7 +56,7 @@ public class ContextManager {
private Map<String,Context> contexts = new HashMap<String,Context>();
- private volatile ContextConfig config;
+ private volatile ContextsConfig config;
private FileSystemManager vfs;
private ReloadingClassLoader parent;
@@ -65,10 +65,28 @@ public class ContextManager {
this.parent = parent;
}
- public interface ContextConfig {
- String getContextURIs(String context);
+ public static class ContextConfig {
+ String uris;
+ boolean preDelegation;
+
+ public ContextConfig(String uris, boolean preDelegation) {
+ this.uris = uris;
+ this.preDelegation = preDelegation;
+ }
- boolean isIsolated(String context);
+ public boolean equals(Object o) {
+ if (o instanceof ContextConfig) {
+ ContextConfig oc = (ContextConfig) o;
+
+ return uris.equals(oc.uris) && preDelegation == oc.preDelegation;
+ }
+
+ return false;
+ }
+ }
+
+ public interface ContextsConfig {
+ ContextConfig getContextConfig(String context);
}
/**
@@ -76,7 +94,7 @@ public class ContextManager {
*
* @param config
*/
- public synchronized void setContextConfig(ContextConfig config) {
+ public synchronized void setContextConfig(ContextsConfig config) {
if (this.config != null)
throw new IllegalStateException("Context manager config already set");
this.config = config;
@@ -84,9 +102,9 @@ public class ContextManager {
public ClassLoader getClassLoader(String contextName) throws
FileSystemException {
- String uris = config.getContextURIs(contextName);
+ ContextConfig cconfig = config.getContextConfig(contextName);
- if (uris == null)
+ if (cconfig == null)
throw new IllegalArgumentException("Unknown context " + contextName);
Context context = null;
@@ -98,11 +116,11 @@ public class ContextManager {
context = contexts.get(context);
if (context == null) {
- context = new Context(uris);
+ context = new Context(cconfig);
contexts.put(contextName, context);
- } else if (!context.uris.equals(uris)) {
+ } else if (!context.cconfig.equals(cconfig)) {
contextToClose = context;
- context = new Context(uris);
+ context = new Context(cconfig);
contexts.put(contextName, context);
}
}
@@ -110,13 +128,13 @@ public class ContextManager {
if (contextToClose != null)
contextToClose.close();
- AccumuloReloadingVFSClassLoader loader = context.getClassLoader();
+ ClassLoader loader = context.getClassLoader();
if (loader == null) {
// ooppss, context was closed by another thread, try again
return getClassLoader(contextName);
}
- return loader.getClassLoader();
+ return loader;
}
Added:
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/PostDelegatingVFSClassLoader.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/PostDelegatingVFSClassLoader.java?rev=1430092&view=auto
==============================================================================
---
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/PostDelegatingVFSClassLoader.java
(added)
+++
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/PostDelegatingVFSClassLoader.java
Mon Jan 7 23:51:06 2013
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.start.classloader.vfs;
+
+import org.apache.commons.vfs2.FileObject;
+import org.apache.commons.vfs2.FileSystemException;
+import org.apache.commons.vfs2.FileSystemManager;
+import org.apache.commons.vfs2.impl.VFSClassLoader;
+
+/**
+ *
+ */
+public class PostDelegatingVFSClassLoader extends VFSClassLoader {
+
+ /**
+ * @param files
+ * @param manager
+ * @param parent
+ * @throws FileSystemException
+ */
+ public PostDelegatingVFSClassLoader(FileObject[] files, FileSystemManager
manager, ClassLoader parent) throws FileSystemException {
+ super(files, manager, parent);
+ }
+
+ protected synchronized Class<?> loadClass(String name, boolean resolve)
throws ClassNotFoundException {
+ Class<?> c = findLoadedClass(name);
+ if (c == null) {
+ try {
+ // try finding this class here instead of parent
+ c = findClass(name);
+ } catch (ClassNotFoundException e) {
+
+ }
+ }
+ return super.loadClass(name, resolve);
+ }
+
+}
Modified:
accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoaderTest.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoaderTest.java?rev=1430092&r1=1430091&r2=1430092&view=diff
==============================================================================
---
accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoaderTest.java
(original)
+++
accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoaderTest.java
Mon Jan 7 23:51:06 2013
@@ -74,7 +74,7 @@ public class AccumuloReloadingVFSClassLo
public ClassLoader getClassLoader() {
return ClassLoader.getSystemClassLoader();
}
- });
+ }, true);
VFSClassLoader cl = (VFSClassLoader) arvcl.getClassLoader();
@@ -94,7 +94,7 @@ public class AccumuloReloadingVFSClassLo
public ClassLoader getClassLoader() {
return ClassLoader.getSystemClassLoader();
}
- }, 1000);
+ }, 1000, true);
FileObject[] files = ((VFSClassLoader)
arvcl.getClassLoader()).getFileObjects();
Assert.assertArrayEquals(createFileSystems(dirContents), files);
Modified:
accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ContextManagerTest.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ContextManagerTest.java?rev=1430092&r1=1430091&r2=1430092&view=diff
==============================================================================
---
accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ContextManagerTest.java
(original)
+++
accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ContextManagerTest.java
Mon Jan 7 23:51:06 2013
@@ -20,6 +20,7 @@ import java.net.URL;
import java.util.HashSet;
import org.apache.accumulo.start.classloader.vfs.ContextManager.ContextConfig;
+import org.apache.accumulo.start.classloader.vfs.ContextManager.ContextsConfig;
import org.apache.accumulo.test.AccumuloDFSBase;
import org.apache.commons.vfs2.FileObject;
import org.apache.commons.vfs2.FileSystemException;
@@ -77,22 +78,16 @@ public class ContextManagerTest extends
}
});
- cm.setContextConfig(new ContextConfig() {
+ cm.setContextConfig(new ContextsConfig() {
@Override
- public String getContextURIs(String context) {
+ public ContextConfig getContextConfig(String context) {
if (context.equals("CX1")) {
- return new Path(TEST_DIR, "HelloWorld.jar").toUri().toString();
+ return new ContextConfig(new Path(TEST_DIR,
"HelloWorld.jar").toUri().toString(), true);
} else if (context.equals("CX2")) {
- return new Path(TEST_DIR2, "HelloWorld.jar").toUri().toString();
+ return new ContextConfig(new Path(TEST_DIR2,
"HelloWorld.jar").toUri().toString(), true);
}
return null;
}
-
- @Override
- public boolean isIsolated(String context) {
- // TODO Auto-generated method stub
- return false;
- }
});
FileObject testDir = vfs.resolveFile(TEST_DIR.toUri().toString());
@@ -120,6 +115,35 @@ public class ContextManagerTest extends
cm.removeUnusedContexts(new HashSet<String>());
}
+ @Test
+ public void testPostDelegation() throws Exception {
+ final VFSClassLoader parent = new VFSClassLoader(new FileObject[]
{vfs.resolveFile(new Path(TEST_DIR, "HelloWorld.jar").toUri().toString())},
vfs);
+
+ Class<?> pclass = parent.loadClass("test.HelloWorld");
+
+ ContextManager cm = new ContextManager(vfs, new ReloadingClassLoader() {
+ @Override
+ public ClassLoader getClassLoader() {
+ return parent;
+ }
+ });
+
+ cm.setContextConfig(new ContextsConfig() {
+ @Override
+ public ContextConfig getContextConfig(String context) {
+ if (context.equals("CX1")) {
+ return new ContextConfig(new Path(TEST_DIR2,
"HelloWorld.jar").toUri().toString(), true);
+ } else if (context.equals("CX2")) {
+ return new ContextConfig(new Path(TEST_DIR2,
"HelloWorld.jar").toUri().toString(), false);
+ }
+ return null;
+ }
+ });
+
+ Assert.assertTrue(cm.getClassLoader("CX1").loadClass("test.HelloWorld") ==
pclass);
+ Assert.assertFalse(cm.getClassLoader("CX2").loadClass("test.HelloWorld")
== pclass);
+ }
+
@After
public void tearDown() throws Exception {
this.hdfs.delete(TEST_DIR, true);
