Author: liyin Date: Sat Apr 26 18:18:32 2014 New Revision: 1590272 URL: http://svn.apache.org/r1590272 Log: [master] Change of CuttingClassLoader and ThrfitResultSerializer/2 for loading different version of Thrift
Author: daviddeng Summary: `instance` in `ThrfitResultSerializer2` is declared as Object and used with reflection `protocolClass` is loaded with same classloader of `ThrfitResultSerializer` Add some logs Test Plan: `TestThrfitResultSerializer` Reviewers: rshroff, manukranthk, liyintang, gauravm, adela Reviewed By: adela CC: hbase-eng@, ssl-diffs@, arice Differential Revision: https://phabricator.fb.com/D1294940 Task ID: 3547474 Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/coprocessor/endpoints/HTableEndpointClient.java hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/thrift/CuttingClassLoader.java hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/thrift/ThriftResultSerializer.java hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/thrift/ThriftResultSerializer2.java hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/ExceptionUtils.java hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftResultSerializer.java Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/coprocessor/endpoints/HTableEndpointClient.java URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/coprocessor/endpoints/HTableEndpointClient.java?rev=1590272&r1=1590271&r2=1590272&view=diff ============================================================================== --- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/coprocessor/endpoints/HTableEndpointClient.java (original) +++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/coprocessor/endpoints/HTableEndpointClient.java Sat Apr 26 18:18:32 2014 @@ -104,7 +104,7 @@ public class HTableEndpointClient implem results.put(region, caller.call(ep)); } } catch (UndeclaredThrowableException e) { - ExceptionUtils.throwIOExcetion(e.getUndeclaredThrowable()); + throw ExceptionUtils.toIOException(e.getUndeclaredThrowable()); } return results; Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/thrift/CuttingClassLoader.java URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/thrift/CuttingClassLoader.java?rev=1590272&r1=1590271&r2=1590272&view=diff ============================================================================== --- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/thrift/CuttingClassLoader.java (original) +++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/thrift/CuttingClassLoader.java Sat Apr 26 18:18:32 2014 @@ -21,32 +21,63 @@ package org.apache.hadoop.hbase.thrift; import java.net.URL; import java.net.URLClassLoader; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; /** * The classloader allows the searching firstly in some path, if not found * search in parent. + * + * The class is useful for loading classes whose dependency is incompatible with + * the current classloader. + * + * skipClasses are set to make those classes to be loaded with parent loader + * and thus can be used by classes loaded by both classloaders. They are + * commonly classes used in the interface. If the class doesn't exist in the + * cutting path, it is not necessary to put in skipClasses. + * + * Class loaded with different classloader is not assignable to each other, use + * reflection to call its methods. */ public class CuttingClassLoader extends URLClassLoader { - public CuttingClassLoader(URL[] urls, ClassLoader parent) { + private static Log LOG = LogFactory.getLog(CuttingClassLoader.class); + private Set<String> skipClasses = new HashSet<>(); + + public CuttingClassLoader(URL[] urls, ClassLoader parent, + String... skipClasses) { super(urls, parent); + Collections.addAll(this.skipClasses, skipClasses); } @Override protected synchronized Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundException { + if (skipClasses.contains(name)) { + // Find in parent + LOG.debug("Finding " + name + " in parent path!"); + return super.loadClass(name, resolve); + } + // Find in loaded classes Class<?> clazz = findLoadedClass(name); if (clazz != null) { + LOG.debug("Found " + name + " in loaded class!"); return clazz; } try { // Find in cutting paths + LOG.debug("Finding " + name + " in cutting path!"); return findClass(name); } catch (ClassNotFoundException e) { } // Find in parent + LOG.debug("Finding " + name + " in parent path!"); return super.loadClass(name, resolve); } } Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/thrift/ThriftResultSerializer.java URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/thrift/ThriftResultSerializer.java?rev=1590272&r1=1590271&r2=1590272&view=diff ============================================================================== --- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/thrift/ThriftResultSerializer.java (original) +++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/thrift/ThriftResultSerializer.java Sat Apr 26 18:18:32 2014 @@ -28,8 +28,8 @@ import org.apache.hadoop.conf.Configurab import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.hbase.mapred.TableInputFormat; import org.apache.hadoop.hbase.thrift.generated.TRowResult; +import org.apache.hadoop.hbase.util.ExceptionUtils; import org.apache.hadoop.io.serializer.Serializer; import org.apache.thrift.TException; import org.apache.thrift.protocol.TCompactProtocol; @@ -38,18 +38,18 @@ import org.apache.thrift.transport.TIOSt import org.apache.thrift.transport.TTransport; /** - * A serializer for use with {@link TableInputFormat}. Serializes results as TResult. + * A serializer for use with TableInputFormat. Serializes results as TResult. */ public class ThriftResultSerializer implements Serializer<Object>, Configurable { private static final Log LOG = LogFactory.getLog(ThriftResultSerializer.class); - public static final String PROTOCOL_CONF_KEY = "hbase.thrift.result.serializer.protocol.class"; - + public static final String PROTOCOL_CONF_KEY = + "hbase.thrift.result.serializer.protocol.class"; + private OutputStream out; - - @SuppressWarnings("rawtypes") - private Class protocolClass = TCompactProtocol.class; + + private Class<? extends TProtocol> protocolClass = TCompactProtocol.class; private TProtocol prot; private DataOutput dataOut; @@ -63,15 +63,15 @@ public class ThriftResultSerializer impl transport = new TIOStreamTransport(out); LOG.info("Using Thrift protocol: " + protocolClass.getName()); - + try { - Constructor<TProtocol> constructor = - protocolClass.getConstructor(new Class[] { TTransport.class }); + Constructor<? extends TProtocol> constructor = + protocolClass.getConstructor(TTransport.class); prot = constructor.newInstance(transport); } catch (Exception ex) { - throw new RuntimeException(ex); + throw ExceptionUtils.toIOException(ex); } - + if (out instanceof DataOutput) { dataOut = (DataOutput) out; } else { @@ -79,10 +79,9 @@ public class ThriftResultSerializer impl } } - @SuppressWarnings("rawtypes") @Override public void serialize(Object t) throws IOException { - Class klass = t.getClass(); + Class<?> klass = t.getClass(); if (klass == Result.class) { Result result = (Result) t; TRowResult tResult = ThriftUtilities.oneRowResult(result); @@ -105,10 +104,21 @@ public class ThriftResultSerializer impl out.close(); } + @SuppressWarnings("unchecked") @Override public void setConf(Configuration conf) { this.conf = conf; - protocolClass = conf.getClass(PROTOCOL_CONF_KEY, protocolClass); + String protoclCLassName = + conf.get(PROTOCOL_CONF_KEY, protocolClass.getName()); + try { + protocolClass = + (Class<? extends TProtocol>) Class.forName(protoclCLassName); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + + LOG.info("Classloader of " + protocolClass.getName() + " is " + + protocolClass.getClassLoader()); } @Override Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/thrift/ThriftResultSerializer2.java URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/thrift/ThriftResultSerializer2.java?rev=1590272&r1=1590271&r2=1590272&view=diff ============================================================================== --- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/thrift/ThriftResultSerializer2.java (original) +++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/thrift/ThriftResultSerializer2.java Sat Apr 26 18:18:32 2014 @@ -22,12 +22,20 @@ package org.apache.hadoop.hbase.thrift; import java.io.File; import java.io.IOException; import java.io.OutputStream; +import java.lang.reflect.Method; import java.net.URL; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.util.ExceptionUtils; import org.apache.hadoop.io.serializer.Serializer; + /** * A wrapper class which loads <code>ThriftResultSerializer</code> class with a * <code>CuttingClassLoader</code>. @@ -37,16 +45,25 @@ import org.apache.hadoop.io.serializer.S */ public class ThriftResultSerializer2 implements Serializer<Object>, Configurable { - ThriftResultSerializer instance; + private static Log LOG = LogFactory.getLog(ThriftResultSerializer2.class); + // ThriftResultSerializer instance; + Object instance; /** * Key to the configuration of classpath to cut */ public static final String CUTTING_CLASSPATH_KEY = "hbase.thrift.result.serializer.cutting.classpath"; + private Method getConfMethod; + private Method openMethod; + private Method serializeMethod; + private Method closeMethod; + + @SuppressWarnings("resource") @Override public void setConf(Configuration conf) { String classPath = conf.get(CUTTING_CLASSPATH_KEY, ""); + LOG.info(CUTTING_CLASSPATH_KEY + " is set to " + classPath); try { // make the URL array of cutting paths String[] parts = classPath.split(File.pathSeparator); @@ -56,37 +73,69 @@ public class ThriftResultSerializer2 imp } // Create the ClassLoader + // Skip all classes in the interface so that both side can use it. ClassLoader classLoader = new CuttingClassLoader(urls, - this.getClass().getClassLoader()); + ThriftResultSerializer.class.getClassLoader(), + Result.class.getName(), ImmutableBytesWritable.class.getName(), + KeyValue.class.getName(), Configuration.class.getName()); // and load the class. - Class<?> cls = Class.forName(ThriftResultSerializer.class.getName(), - true, classLoader); + Class<?> cls = + classLoader.loadClass(ThriftResultSerializer.class.getName()); // Then create the new instance. - instance = (ThriftResultSerializer) cls.newInstance(); + instance = cls.newInstance(); } catch (Exception e) { - throw new RuntimeException(e); + throw ExceptionUtils.toRuntimeException(e); } - instance.setConf(conf); + try { + getConfMethod = instance.getClass().getMethod("getConf"); + openMethod = instance.getClass().getMethod("open", OutputStream.class); + serializeMethod = + instance.getClass().getMethod("serialize", Object.class); + closeMethod = instance.getClass().getMethod("close"); + + Method setConfMethod = + instance.getClass().getMethod("setConf", Configuration.class); + setConfMethod.invoke(instance, conf); + } catch (Exception e) { + throw ExceptionUtils.toRuntimeException(e); + } } @Override public Configuration getConf() { - return instance.getConf(); + try { + return (Configuration) getConfMethod.invoke(instance); + } catch (Exception e) { + throw ExceptionUtils.toRuntimeException(e); + } } @Override public void open(OutputStream out) throws IOException { - instance.open(out); + try { + openMethod.invoke(instance, out); + } catch (Exception e) { + throw ExceptionUtils.toRuntimeException(e); + } } @Override public void serialize(Object t) throws IOException { - instance.serialize(t); + try { + serializeMethod.invoke(instance, t); + } catch (Exception e) { + throw ExceptionUtils.toRuntimeException(e); + } } @Override public void close() throws IOException { - instance.close(); + try { + closeMethod.invoke(instance); + } catch (Exception e) { + throw ExceptionUtils.toRuntimeException(e); + } } + } Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/ExceptionUtils.java URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/ExceptionUtils.java?rev=1590272&r1=1590271&r2=1590272&view=diff ============================================================================== --- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/ExceptionUtils.java (original) +++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/ExceptionUtils.java Sat Apr 26 18:18:32 2014 @@ -20,25 +20,70 @@ package org.apache.hadoop.hbase.util; import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.UndeclaredThrowableException; /** * Some utility methods for handling exception. */ public class ExceptionUtils { /** - * Throws an arbitrary exception to an IOException. Wrap the exception with - * an IOException if the exception cannot be thrown directly. + * If t is an RuntimeException throw it directly, or + * If t is an InvocationTargetException, recursively call toIOException + * with target exception, or + * If t is an UndeclaredThrowableException, recursively call toIOException + * with undeclared throwable, or + * Otherwise convert it into an IOException. Wrap the exception with an + * IOException if necessary. + * + * DO NOT use it if you want to wrap all exception including RuntimeException + * into an IOException. */ - public static void throwIOExcetion(Throwable t) - throws IOException { + public static IOException toIOException(Throwable t) { + if (t instanceof InvocationTargetException) { + return toIOException(((InvocationTargetException) t) + .getTargetException()); + } + + if (t instanceof UndeclaredThrowableException) { + return toIOException(((UndeclaredThrowableException) t) + .getUndeclaredThrowable()); + } + if (t instanceof IOException) { - throw (IOException) t; + return (IOException) t; } if (t instanceof RuntimeException) { throw (RuntimeException) t; } - throw new IOException(t); + return new IOException(t); + } + + /** + * If t is an RuntimeException return it directly, or + * If t is an InvocationTargetException, recursively call toRuntimeException + * with target exception, or + * If t is an UndeclaredThrowableException, recursively call + * toRuntimeException with undeclared throwable, or + * Otherwise wrap it into an toRuntimeException. + */ + public static RuntimeException toRuntimeException(Throwable t) { + if (t instanceof InvocationTargetException) { + return toRuntimeException(((InvocationTargetException) t) + .getTargetException()); + } + + if (t instanceof UndeclaredThrowableException) { + return toRuntimeException(((UndeclaredThrowableException) t) + .getUndeclaredThrowable()); + } + + if (t instanceof RuntimeException) { + return (RuntimeException) t; + } + + return new RuntimeException(t); } } Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftResultSerializer.java URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftResultSerializer.java?rev=1590272&r1=1590271&r2=1590272&view=diff ============================================================================== --- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftResultSerializer.java (original) +++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftResultSerializer.java Sat Apr 26 18:18:32 2014 @@ -20,9 +20,6 @@ package org.apache.hadoop.hbase.thrift; import java.io.ByteArrayOutputStream; -import java.io.File; -import java.util.ArrayList; -import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; @@ -38,15 +35,15 @@ import org.junit.Test; */ public class TestThriftResultSerializer { - private KeyValue[] constuctKvList(int n) { - List<KeyValue> list = new ArrayList<KeyValue>(); + private static KeyValue[] constuctKvList(int n) { + KeyValue[] kvs = new KeyValue[n]; for (int i = 0; i < n; i++) { KeyValue kv = new KeyValue(Bytes.toBytes("myRow" + i), Bytes.toBytes("myCF"), Bytes.toBytes("myQualifier"), 12345L, Bytes.toBytes("myValue")); - list.add(kv); + kvs[i] = kv; } - return list.toArray(new KeyValue[list.size()]); + return kvs; } @Test @@ -69,28 +66,19 @@ public class TestThriftResultSerializer Assert.assertTrue("output is empty", out.size() > 0); } - private static String findThriftPath() { - String[] paths = System.getProperty("java.class.path").split( - File.pathSeparator); - for (String path : paths) { - if (path.contains("/libthrift-")) { - return path; - } - } - return ""; - } - @Test public void testWithClassPath() throws Exception { ByteArrayOutputStream out = new ByteArrayOutputStream(); ThriftResultSerializer2 trs = new ThriftResultSerializer2(); - String cp = findThriftPath(); + String cp = System.getProperty("java.class.path"); System.out.println("Cutting classpath: " + cp); Configuration conf = HBaseConfiguration.create(); conf.set(ThriftResultSerializer2.CUTTING_CLASSPATH_KEY, cp); + conf.set(ThriftResultSerializer.PROTOCOL_CONF_KEY, + "org.apache.thrift.protocol.TCompactProtocol"); trs.setConf(conf); trs.open(out); @@ -103,4 +91,33 @@ public class TestThriftResultSerializer Assert.assertTrue("output is empty", out.size() > 0); } + + public static void main(String[] args) throws Exception { + System.out.println("java.class.path: " + + System.getProperty("java.class.path")); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ThriftResultSerializer2 trs = new ThriftResultSerializer2(); + + Configuration conf = HBaseConfiguration.create(); + if (args.length >= 1) { + System.out.println("Setting " + + ThriftResultSerializer2.CUTTING_CLASSPATH_KEY + " to " + args[0]); + conf.set(ThriftResultSerializer2.CUTTING_CLASSPATH_KEY, args[0]); + } + if (args.length >= 2) { + System.out.println("Setting " + ThriftResultSerializer.PROTOCOL_CONF_KEY + + " to " + args[1]); + conf.set(ThriftResultSerializer.PROTOCOL_CONF_KEY, args[1]); + } + trs.setConf(conf); + + trs.open(out); + try { + trs.serialize(new ImmutableBytesWritable(Bytes.toBytes("row"))); + trs.serialize(new Result(constuctKvList(10))); + } finally { + trs.close(); + } + } }
