[03/32] git commit: ACCUMULO-2831: Reset the classloader on the Hadoop Configuration objects that were created before the VFS ClassLoader was created.

2014-05-28 Thread elserj 
ACCUMULO-2831: Reset the classloader on the Hadoop Configuration objects that 
were created before the VFS ClassLoader was created.


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/db1b63c4
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/db1b63c4
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/db1b63c4

Branch: refs/heads/ACCUMULO-378
Commit: db1b63c4cb9f2f1bfdb45765b9c79f84105d601a
Parents: 1193f4b
Author: Dave Marion dlmar...@hotmail.com
Authored: Wed May 21 21:11:26 2014 -0400
Committer: Dave Marion dlmar...@hotmail.com
Committed: Wed May 21 21:11:26 2014 -0400

--
 .../classloader/vfs/AccumuloVFSClassLoader.java  | 19 +++
 1 file changed, 19 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/accumulo/blob/db1b63c4/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
--
diff --git 
a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
 
b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
index fa20725..d4044a5 100644
--- 
a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
+++ 
b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
@@ -27,6 +27,7 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.accumulo.start.classloader.AccumuloClassLoader;
+import org.apache.accumulo.start.classloader.vfs.providers.HdfsFileObject;
 import org.apache.accumulo.start.classloader.vfs.providers.HdfsFileProvider;
 import org.apache.commons.vfs2.CacheStrategy;
 import org.apache.commons.vfs2.FileObject;
@@ -37,6 +38,8 @@ import org.apache.commons.vfs2.cache.SoftRefFilesCache;
 import org.apache.commons.vfs2.impl.DefaultFileSystemManager;
 import org.apache.commons.vfs2.impl.FileContentInfoFilenameFactory;
 import org.apache.commons.vfs2.impl.VFSClassLoader;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.log4j.Logger;
 
 /**
@@ -213,6 +216,22 @@ public class AccumuloVFSClassLoader {
   // Create the Accumulo Context ClassLoader using the DEFAULT_CONTEXT
   localLoader = createDynamicClassloader(new VFSClassLoader(vfsCP, 
vfs, parent));
   loader = localLoader;
+
+  //An HDFS FileSystem and Configuration object were created for each 
unique HDFS namespace in the call to resolve above.
+  //The HDFS Client did us a favor and cached these objects so that 
the next time someone calls FileSystem.get(uri), they
+  //get the cached object. However, these objects were created not 
with the system VFS classloader, but the classloader above
+  //it. We need to override the classloader on the Configuration 
objects. Ran into an issue were log recovery was being attempted
+  //and SequenceFile$Reader was trying to instantiate the key class 
via WritableName.getClass(String, Configuration)
+  for (FileObject fo : vfsCP) {
+if (fo instanceof HdfsFileObject) {
+  String uri = fo.getName().getRootURI();
+  Configuration c = new Configuration(true);
+  c.set(FileSystem.FS_DEFAULT_NAME_KEY, uri);
+  FileSystem fs = FileSystem.get(c);
+  fs.getConf().setClassLoader(loader.getClassLoader());
+}
+  }
+
 }
   }
 }

git commit: ACCUMULO-2831: Reset the classloader on the Hadoop Configuration objects that were created before the VFS ClassLoader was created.

2014-05-21 Thread dlmarion 
Repository: accumulo
Updated Branches:
  refs/heads/1.6.1-SNAPSHOT 1193f4b84 - db1b63c4c


ACCUMULO-2831: Reset the classloader on the Hadoop Configuration objects that 
were created before the VFS ClassLoader was created.


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/db1b63c4
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/db1b63c4
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/db1b63c4

Branch: refs/heads/1.6.1-SNAPSHOT
Commit: db1b63c4cb9f2f1bfdb45765b9c79f84105d601a
Parents: 1193f4b
Author: Dave Marion dlmar...@hotmail.com
Authored: Wed May 21 21:11:26 2014 -0400
Committer: Dave Marion dlmar...@hotmail.com
Committed: Wed May 21 21:11:26 2014 -0400

--
 .../classloader/vfs/AccumuloVFSClassLoader.java  | 19 +++
 1 file changed, 19 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/accumulo/blob/db1b63c4/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
--
diff --git 
a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
 
b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
index fa20725..d4044a5 100644
--- 
a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
+++ 
b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
@@ -27,6 +27,7 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.accumulo.start.classloader.AccumuloClassLoader;
+import org.apache.accumulo.start.classloader.vfs.providers.HdfsFileObject;
 import org.apache.accumulo.start.classloader.vfs.providers.HdfsFileProvider;
 import org.apache.commons.vfs2.CacheStrategy;
 import org.apache.commons.vfs2.FileObject;
@@ -37,6 +38,8 @@ import org.apache.commons.vfs2.cache.SoftRefFilesCache;
 import org.apache.commons.vfs2.impl.DefaultFileSystemManager;
 import org.apache.commons.vfs2.impl.FileContentInfoFilenameFactory;
 import org.apache.commons.vfs2.impl.VFSClassLoader;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.log4j.Logger;
 
 /**
@@ -213,6 +216,22 @@ public class AccumuloVFSClassLoader {
   // Create the Accumulo Context ClassLoader using the DEFAULT_CONTEXT
   localLoader = createDynamicClassloader(new VFSClassLoader(vfsCP, 
vfs, parent));
   loader = localLoader;
+
+  //An HDFS FileSystem and Configuration object were created for each 
unique HDFS namespace in the call to resolve above.
+  //The HDFS Client did us a favor and cached these objects so that 
the next time someone calls FileSystem.get(uri), they
+  //get the cached object. However, these objects were created not 
with the system VFS classloader, but the classloader above
+  //it. We need to override the classloader on the Configuration 
objects. Ran into an issue were log recovery was being attempted
+  //and SequenceFile$Reader was trying to instantiate the key class 
via WritableName.getClass(String, Configuration)
+  for (FileObject fo : vfsCP) {
+if (fo instanceof HdfsFileObject) {
+  String uri = fo.getName().getRootURI();
+  Configuration c = new Configuration(true);
+  c.set(FileSystem.FS_DEFAULT_NAME_KEY, uri);
+  FileSystem fs = FileSystem.get(c);
+  fs.getConf().setClassLoader(loader.getClassLoader());
+}
+  }
+
 }
   }
 }

[1/2] git commit: ACCUMULO-2831: Reset the classloader on the Hadoop Configuration objects that were created before the VFS ClassLoader was created.

2014-05-21 Thread dlmarion 
Repository: accumulo
Updated Branches:
  refs/heads/master f985d3c3b - 58e7665ae


ACCUMULO-2831: Reset the classloader on the Hadoop Configuration objects that 
were created before the VFS ClassLoader was created.


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/db1b63c4
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/db1b63c4
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/db1b63c4

Branch: refs/heads/master
Commit: db1b63c4cb9f2f1bfdb45765b9c79f84105d601a
Parents: 1193f4b
Author: Dave Marion dlmar...@hotmail.com
Authored: Wed May 21 21:11:26 2014 -0400
Committer: Dave Marion dlmar...@hotmail.com
Committed: Wed May 21 21:11:26 2014 -0400

--
 .../classloader/vfs/AccumuloVFSClassLoader.java  | 19 +++
 1 file changed, 19 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/accumulo/blob/db1b63c4/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
--
diff --git 
a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
 
b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
index fa20725..d4044a5 100644
--- 
a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
+++ 
b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
@@ -27,6 +27,7 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.accumulo.start.classloader.AccumuloClassLoader;
+import org.apache.accumulo.start.classloader.vfs.providers.HdfsFileObject;
 import org.apache.accumulo.start.classloader.vfs.providers.HdfsFileProvider;
 import org.apache.commons.vfs2.CacheStrategy;
 import org.apache.commons.vfs2.FileObject;
@@ -37,6 +38,8 @@ import org.apache.commons.vfs2.cache.SoftRefFilesCache;
 import org.apache.commons.vfs2.impl.DefaultFileSystemManager;
 import org.apache.commons.vfs2.impl.FileContentInfoFilenameFactory;
 import org.apache.commons.vfs2.impl.VFSClassLoader;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.log4j.Logger;
 
 /**
@@ -213,6 +216,22 @@ public class AccumuloVFSClassLoader {
   // Create the Accumulo Context ClassLoader using the DEFAULT_CONTEXT
   localLoader = createDynamicClassloader(new VFSClassLoader(vfsCP, 
vfs, parent));
   loader = localLoader;
+
+  //An HDFS FileSystem and Configuration object were created for each 
unique HDFS namespace in the call to resolve above.
+  //The HDFS Client did us a favor and cached these objects so that 
the next time someone calls FileSystem.get(uri), they
+  //get the cached object. However, these objects were created not 
with the system VFS classloader, but the classloader above
+  //it. We need to override the classloader on the Configuration 
objects. Ran into an issue were log recovery was being attempted
+  //and SequenceFile$Reader was trying to instantiate the key class 
via WritableName.getClass(String, Configuration)
+  for (FileObject fo : vfsCP) {
+if (fo instanceof HdfsFileObject) {
+  String uri = fo.getName().getRootURI();
+  Configuration c = new Configuration(true);
+  c.set(FileSystem.FS_DEFAULT_NAME_KEY, uri);
+  FileSystem fs = FileSystem.get(c);
+  fs.getConf().setClassLoader(loader.getClassLoader());
+}
+  }
+
 }
   }
 }