This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/master by this push:
new 18a9221 Add retry mechanism to AccumuloReloadingVFSClassLoader (#453)
new b484841 Merge branch 'master' of github.com:apache/accumulo
18a9221 is described below
commit 18a9221581c7676f748899b2a274ce04f71600f9
Author: Hannah Pellon <>
AuthorDate: Mon Apr 30 18:51:10 2018 +0000
Add retry mechanism to AccumuloReloadingVFSClassLoader (#453)
* Also addresses issues mentioned in ACCUMULO-1507
---
.../vfs/AccumuloReloadingVFSClassLoader.java | 54 +++++++++++++++++++++-
.../classloader/vfs/AccumuloVFSClassLoader.java | 17 ++++---
.../vfs/AccumuloReloadingVFSClassLoaderTest.java | 54 ++++++++++++++++++++++
3 files changed, 118 insertions(+), 7 deletions(-)
diff --git
a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java
b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java
index 0f275c9..a084db3 100644
---
a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java
+++
b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java
@@ -45,10 +45,16 @@ public class AccumuloReloadingVFSClassLoader implements
FileListener, ReloadingC
private static final Logger log =
LoggerFactory.getLogger(AccumuloReloadingVFSClassLoader.class);
- // set to 5 mins. The rational behind this large time is to avoid a
gazillion tservers all asking
+ // set to 5 mins. The rationale behind this large time is to avoid a
gazillion tservers all asking
// the name node for info too frequently.
private static final int DEFAULT_TIMEOUT = 5 * 60 * 1000;
+ private volatile long maxWaitInterval = 60000;
+
+ private volatile long maxRetries = -1;
+
+ private volatile long sleepInterval = 5000;
+
private FileObject[] files;
private VFSClassLoader cl;
private final ReloadingClassLoader parent;
@@ -79,6 +85,35 @@ public class AccumuloReloadingVFSClassLoader implements
FileListener, ReloadingC
FileSystemManager vfs = AccumuloVFSClassLoader.generateVfs();
FileObject[] files = AccumuloVFSClassLoader.resolve(vfs, uris);
+ long retries = 0;
+ long currentSleepMillis = sleepInterval;
+
+ if (files.length == 0) {
+ while (files.length == 0 && retryPermitted(retries)) {
+
+ try {
+ log.debug("VFS path was empty. Waiting " + currentSleepMillis
+ " ms to retry");
+ Thread.sleep(currentSleepMillis);
+
+ files = AccumuloVFSClassLoader.resolve(vfs, uris);
+ retries++;
+
+ currentSleepMillis = Math.min(maxWaitInterval,
currentSleepMillis + sleepInterval);
+
+ } catch (InterruptedException e) {
+ log.error("VFS Retry Interruped", e);
+ throw new RuntimeException(e);
+ }
+
+ }
+ // There is a chance that the listener was removed from the top
level directory or
+ // its children if they were deleted within some time window.
Re-add files to be
+ // monitored. The Monitor will ignore files that are already/still
being monitored.
+ for (FileObject file : files) {
+ monitor.addFile(file);
+ }
+ }
+
log.debug("Rebuilding dynamic classloader using files- {}",
stringify(files));
VFSClassLoader cl;
@@ -214,4 +249,21 @@ public class AccumuloReloadingVFSClassLoader implements
FileListener, ReloadingC
return buf.toString();
}
+ // VisibleForTesting intentionally not using annotation from Guava because
it adds unwanted
+ // dependency
+ void setMaxRetries(long maxRetries) {
+ this.maxRetries = maxRetries;
+ }
+
+ long getMaxRetries() {
+ return maxRetries;
+ }
+
+ long getMaxWaitInterval() {
+ return maxWaitInterval;
+ }
+
+ private boolean retryPermitted(long retries) {
+ return (maxRetries < 0 || retries < maxRetries);
+ }
}
diff --git
a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
index 7ee586c..3aa87eb 100644
---
a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
+++
b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
@@ -151,14 +151,19 @@ public class AccumuloVFSClassLoader {
case IMAGINARY:
// assume its a pattern
String pattern = fo.getName().getBaseName();
- if (fo.getParent() != null && fo.getParent().getType() ==
FileType.FOLDER) {
+ if (fo.getParent() != null) {
+ // still monitor the parent
pathsToMonitor.add(fo.getParent());
- FileObject[] children = fo.getParent().getChildren();
- for (FileObject child : children) {
- if (child.getType() == FileType.FILE
- && child.getName().getBaseName().matches(pattern)) {
- classpath.add(child);
+ if (fo.getParent().getType() == FileType.FOLDER) {
+ FileObject[] children = fo.getParent().getChildren();
+ for (FileObject child : children) {
+ if (child.getType() == FileType.FILE
+ && child.getName().getBaseName().matches(pattern)) {
+ classpath.add(child);
+ }
}
+ } else {
+ log.debug("classpath entry " + fo.getParent() + " is " +
fo.getParent().getType());
}
} else {
log.warn("ignoring classpath entry {}", fo);
diff --git
a/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoaderTest.java
b/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoaderTest.java
index e23130a..92568c9 100644
---
a/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoaderTest.java
+++
b/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoaderTest.java
@@ -103,6 +103,9 @@ public class AccumuloReloadingVFSClassLoaderTest {
FileObject[] files = ((VFSClassLoader)
arvcl.getClassLoader()).getFileObjects();
Assert.assertArrayEquals(createFileSystems(dirContents), files);
+ // set retry settings sufficiently low that not everything is reloaded in
the first round
+ arvcl.setMaxRetries(1);
+
Class<?> clazz1 = arvcl.getClassLoader().loadClass("test.HelloWorld");
Object o1 = clazz1.newInstance();
Assert.assertEquals("Hello World!", o1.toString());
@@ -135,6 +138,57 @@ public class AccumuloReloadingVFSClassLoaderTest {
arvcl.close();
}
+ @Test
+ public void testReloadingWithLongerTimeout() throws Exception {
+ FileObject testDir = vfs.resolveFile(folder1.getRoot().toURI().toString());
+ FileObject[] dirContents = testDir.getChildren();
+
+ AccumuloReloadingVFSClassLoader arvcl = new
AccumuloReloadingVFSClassLoader(folderPath, vfs,
+ new ReloadingClassLoader() {
+ @Override
+ public ClassLoader getClassLoader() {
+ return ClassLoader.getSystemClassLoader();
+ }
+ }, 1000, true);
+
+ FileObject[] files = ((VFSClassLoader)
arvcl.getClassLoader()).getFileObjects();
+ Assert.assertArrayEquals(createFileSystems(dirContents), files);
+
+ // set retry settings sufficiently high such that reloading happens in the
first rounds
+ arvcl.setMaxRetries(3);
+
+ Class<?> clazz1 = arvcl.getClassLoader().loadClass("test.HelloWorld");
+ Object o1 = clazz1.newInstance();
+ Assert.assertEquals("Hello World!", o1.toString());
+
+ // Check that the class is the same before the update
+ Class<?> clazz1_5 = arvcl.getClassLoader().loadClass("test.HelloWorld");
+ Assert.assertEquals(clazz1, clazz1_5);
+
+ assertTrue(new File(folder1.getRoot(), "HelloWorld.jar").delete());
+
+ // VFS-487 significantly wait to avoid failure
+ Thread.sleep(7000);
+
+ // Update the class
+ FileUtils.copyURLToFile(this.getClass().getResource("/HelloWorld.jar"),
+ folder1.newFile("HelloWorld2.jar"));
+
+ // Wait for the monitor to notice
+ // VFS-487 significantly wait to avoid failure
+ Thread.sleep(7000);
+
+ Class<?> clazz2 = arvcl.getClassLoader().loadClass("test.HelloWorld");
+ Object o2 = clazz2.newInstance();
+ Assert.assertEquals("Hello World!", o2.toString());
+
+ // This is true because they are loaded by the same classloader due to the
new retry
+ Assert.assertTrue(clazz1.equals(clazz2));
+ Assert.assertFalse(o1.equals(o2));
+
+ arvcl.close();
+ }
+
// This test fails because of an error with the underlying monitor
(ACCUMULO-1507/VFS-487).
// Uncomment when this has been addressed.
//
--
To stop receiving notification emails like this one, please contact
ktur...@apache.org.
