http://git-wip-us.apache.org/repos/asf/hbase-site/blob/e10887fe/devapidocs/src-html/org/apache/hadoop/hbase/util/CommonFSUtils.StreamCapabilities.html ---------------------------------------------------------------------- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/util/CommonFSUtils.StreamCapabilities.html b/devapidocs/src-html/org/apache/hadoop/hbase/util/CommonFSUtils.StreamCapabilities.html index c6bfdca..cae3d07 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/util/CommonFSUtils.StreamCapabilities.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/util/CommonFSUtils.StreamCapabilities.html @@ -829,90 +829,217 @@ <span class="sourceLineNo">821</span> conf.setIfUnset(dfsKey, Integer.toString(hbaseSize));<a name="line.821"></a> <span class="sourceLineNo">822</span> }<a name="line.822"></a> <span class="sourceLineNo">823</span><a name="line.823"></a> -<span class="sourceLineNo">824</span> // Holder singleton idiom. JVM spec ensures this will be run at most once per Classloader, and<a name="line.824"></a> -<span class="sourceLineNo">825</span> // not until we attempt to reference it.<a name="line.825"></a> -<span class="sourceLineNo">826</span> private static class StreamCapabilities {<a name="line.826"></a> -<span class="sourceLineNo">827</span> public static final boolean PRESENT;<a name="line.827"></a> -<span class="sourceLineNo">828</span> public static final Class<?> CLASS;<a name="line.828"></a> -<span class="sourceLineNo">829</span> public static final Method METHOD;<a name="line.829"></a> -<span class="sourceLineNo">830</span> static {<a name="line.830"></a> -<span class="sourceLineNo">831</span> boolean tmp = false;<a name="line.831"></a> -<span class="sourceLineNo">832</span> Class<?> clazz = null;<a name="line.832"></a> -<span class="sourceLineNo">833</span> Method method = null;<a name="line.833"></a> -<span class="sourceLineNo">834</span> try {<a name="line.834"></a> -<span class="sourceLineNo">835</span> clazz = Class.forName("org.apache.hadoop.fs.StreamCapabilities");<a name="line.835"></a> -<span class="sourceLineNo">836</span> method = clazz.getMethod("hasCapability", String.class);<a name="line.836"></a> -<span class="sourceLineNo">837</span> tmp = true;<a name="line.837"></a> -<span class="sourceLineNo">838</span> } catch(ClassNotFoundException|NoSuchMethodException|SecurityException exception) {<a name="line.838"></a> -<span class="sourceLineNo">839</span> LOG.warn("Your Hadoop installation does not include the StreamCapabilities class from " +<a name="line.839"></a> -<span class="sourceLineNo">840</span> "HDFS-11644, so we will skip checking if any FSDataOutputStreams actually " +<a name="line.840"></a> -<span class="sourceLineNo">841</span> "support hflush/hsync. If you are running on top of HDFS this probably just " +<a name="line.841"></a> -<span class="sourceLineNo">842</span> "means you have an older version and this can be ignored. If you are running on " +<a name="line.842"></a> -<span class="sourceLineNo">843</span> "top of an alternate FileSystem implementation you should manually verify that " +<a name="line.843"></a> -<span class="sourceLineNo">844</span> "hflush and hsync are implemented; otherwise you risk data loss and hard to " +<a name="line.844"></a> -<span class="sourceLineNo">845</span> "diagnose errors when our assumptions are violated.");<a name="line.845"></a> -<span class="sourceLineNo">846</span> LOG.debug("The first request to check for StreamCapabilities came from this stacktrace.",<a name="line.846"></a> -<span class="sourceLineNo">847</span> exception);<a name="line.847"></a> -<span class="sourceLineNo">848</span> } finally {<a name="line.848"></a> -<span class="sourceLineNo">849</span> PRESENT = tmp;<a name="line.849"></a> -<span class="sourceLineNo">850</span> CLASS = clazz;<a name="line.850"></a> -<span class="sourceLineNo">851</span> METHOD = method;<a name="line.851"></a> -<span class="sourceLineNo">852</span> }<a name="line.852"></a> -<span class="sourceLineNo">853</span> }<a name="line.853"></a> -<span class="sourceLineNo">854</span> }<a name="line.854"></a> -<span class="sourceLineNo">855</span><a name="line.855"></a> -<span class="sourceLineNo">856</span> /**<a name="line.856"></a> -<span class="sourceLineNo">857</span> * If our FileSystem version includes the StreamCapabilities class, check if<a name="line.857"></a> -<span class="sourceLineNo">858</span> * the given stream has a particular capability.<a name="line.858"></a> -<span class="sourceLineNo">859</span> * @param stream capabilities are per-stream instance, so check this one specifically. must not be<a name="line.859"></a> -<span class="sourceLineNo">860</span> * null<a name="line.860"></a> -<span class="sourceLineNo">861</span> * @param capability what to look for, per Hadoop Common's FileSystem docs<a name="line.861"></a> -<span class="sourceLineNo">862</span> * @return true if there are no StreamCapabilities. false if there are, but this stream doesn't<a name="line.862"></a> -<span class="sourceLineNo">863</span> * implement it. return result of asking the stream otherwise.<a name="line.863"></a> -<span class="sourceLineNo">864</span> */<a name="line.864"></a> -<span class="sourceLineNo">865</span> public static boolean hasCapability(FSDataOutputStream stream, String capability) {<a name="line.865"></a> -<span class="sourceLineNo">866</span> // be consistent whether or not StreamCapabilities is present<a name="line.866"></a> -<span class="sourceLineNo">867</span> if (stream == null) {<a name="line.867"></a> -<span class="sourceLineNo">868</span> throw new NullPointerException("stream parameter must not be null.");<a name="line.868"></a> -<span class="sourceLineNo">869</span> }<a name="line.869"></a> -<span class="sourceLineNo">870</span> // If o.a.h.fs.StreamCapabilities doesn't exist, assume everyone does everything<a name="line.870"></a> -<span class="sourceLineNo">871</span> // otherwise old versions of Hadoop will break.<a name="line.871"></a> -<span class="sourceLineNo">872</span> boolean result = true;<a name="line.872"></a> -<span class="sourceLineNo">873</span> if (StreamCapabilities.PRESENT) {<a name="line.873"></a> -<span class="sourceLineNo">874</span> // if StreamCapabilities is present, but the stream doesn't implement it<a name="line.874"></a> -<span class="sourceLineNo">875</span> // or we run into a problem invoking the method,<a name="line.875"></a> -<span class="sourceLineNo">876</span> // we treat that as equivalent to not declaring anything<a name="line.876"></a> -<span class="sourceLineNo">877</span> result = false;<a name="line.877"></a> -<span class="sourceLineNo">878</span> if (StreamCapabilities.CLASS.isAssignableFrom(stream.getClass())) {<a name="line.878"></a> +<span class="sourceLineNo">824</span> private static class DfsBuilderUtility {<a name="line.824"></a> +<span class="sourceLineNo">825</span> static Class<?> dfsClass = null;<a name="line.825"></a> +<span class="sourceLineNo">826</span> static Method createMethod;<a name="line.826"></a> +<span class="sourceLineNo">827</span> static Method overwriteMethod;<a name="line.827"></a> +<span class="sourceLineNo">828</span> static Method bufferSizeMethod;<a name="line.828"></a> +<span class="sourceLineNo">829</span> static Method blockSizeMethod;<a name="line.829"></a> +<span class="sourceLineNo">830</span> static Method recursiveMethod;<a name="line.830"></a> +<span class="sourceLineNo">831</span> static Method replicateMethod;<a name="line.831"></a> +<span class="sourceLineNo">832</span> static Method replicationMethod;<a name="line.832"></a> +<span class="sourceLineNo">833</span> static Method buildMethod;<a name="line.833"></a> +<span class="sourceLineNo">834</span> static boolean allMethodsPresent = false;<a name="line.834"></a> +<span class="sourceLineNo">835</span><a name="line.835"></a> +<span class="sourceLineNo">836</span> static {<a name="line.836"></a> +<span class="sourceLineNo">837</span> String dfsName = "org.apache.hadoop.hdfs.DistributedFileSystem";<a name="line.837"></a> +<span class="sourceLineNo">838</span> String builderName = dfsName + "$HdfsDataOutputStreamBuilder";<a name="line.838"></a> +<span class="sourceLineNo">839</span> Class<?> builderClass = null;<a name="line.839"></a> +<span class="sourceLineNo">840</span><a name="line.840"></a> +<span class="sourceLineNo">841</span> try {<a name="line.841"></a> +<span class="sourceLineNo">842</span> dfsClass = Class.forName(dfsName);<a name="line.842"></a> +<span class="sourceLineNo">843</span> } catch (ClassNotFoundException e) {<a name="line.843"></a> +<span class="sourceLineNo">844</span> LOG.debug("{} not available, will not use builder API for file creation.", dfsName);<a name="line.844"></a> +<span class="sourceLineNo">845</span> }<a name="line.845"></a> +<span class="sourceLineNo">846</span> try {<a name="line.846"></a> +<span class="sourceLineNo">847</span> builderClass = Class.forName(builderName);<a name="line.847"></a> +<span class="sourceLineNo">848</span> } catch (ClassNotFoundException e) {<a name="line.848"></a> +<span class="sourceLineNo">849</span> LOG.debug("{} not available, will not use builder API for file creation.", builderName);<a name="line.849"></a> +<span class="sourceLineNo">850</span> }<a name="line.850"></a> +<span class="sourceLineNo">851</span><a name="line.851"></a> +<span class="sourceLineNo">852</span> if (dfsClass != null && builderClass != null) {<a name="line.852"></a> +<span class="sourceLineNo">853</span> try {<a name="line.853"></a> +<span class="sourceLineNo">854</span> createMethod = dfsClass.getMethod("createFile", Path.class);<a name="line.854"></a> +<span class="sourceLineNo">855</span> overwriteMethod = builderClass.getMethod("overwrite", boolean.class);<a name="line.855"></a> +<span class="sourceLineNo">856</span> bufferSizeMethod = builderClass.getMethod("bufferSize", int.class);<a name="line.856"></a> +<span class="sourceLineNo">857</span> blockSizeMethod = builderClass.getMethod("blockSize", long.class);<a name="line.857"></a> +<span class="sourceLineNo">858</span> recursiveMethod = builderClass.getMethod("recursive");<a name="line.858"></a> +<span class="sourceLineNo">859</span> replicateMethod = builderClass.getMethod("replicate");<a name="line.859"></a> +<span class="sourceLineNo">860</span> replicationMethod = builderClass.getMethod("replication", short.class);<a name="line.860"></a> +<span class="sourceLineNo">861</span> buildMethod = builderClass.getMethod("build");<a name="line.861"></a> +<span class="sourceLineNo">862</span><a name="line.862"></a> +<span class="sourceLineNo">863</span> allMethodsPresent = true;<a name="line.863"></a> +<span class="sourceLineNo">864</span> LOG.debug("Using builder API via reflection for DFS file creation.");<a name="line.864"></a> +<span class="sourceLineNo">865</span> } catch (NoSuchMethodException e) {<a name="line.865"></a> +<span class="sourceLineNo">866</span> LOG.debug("Could not find method on builder; will use old DFS API for file creation {}",<a name="line.866"></a> +<span class="sourceLineNo">867</span> e.getMessage());<a name="line.867"></a> +<span class="sourceLineNo">868</span> }<a name="line.868"></a> +<span class="sourceLineNo">869</span> }<a name="line.869"></a> +<span class="sourceLineNo">870</span> }<a name="line.870"></a> +<span class="sourceLineNo">871</span><a name="line.871"></a> +<span class="sourceLineNo">872</span> /**<a name="line.872"></a> +<span class="sourceLineNo">873</span> * Attempt to use builder API via reflection to create a file with the given parameters and<a name="line.873"></a> +<span class="sourceLineNo">874</span> * replication enabled.<a name="line.874"></a> +<span class="sourceLineNo">875</span> */<a name="line.875"></a> +<span class="sourceLineNo">876</span> static FSDataOutputStream createHelper(FileSystem fs, Path path, boolean overwritable,<a name="line.876"></a> +<span class="sourceLineNo">877</span> int bufferSize, short replication, long blockSize, boolean isRecursive) throws IOException {<a name="line.877"></a> +<span class="sourceLineNo">878</span> if (allMethodsPresent && dfsClass.isInstance(fs)) {<a name="line.878"></a> <span class="sourceLineNo">879</span> try {<a name="line.879"></a> -<span class="sourceLineNo">880</span> result = ((Boolean)StreamCapabilities.METHOD.invoke(stream, capability)).booleanValue();<a name="line.880"></a> -<span class="sourceLineNo">881</span> } catch (IllegalAccessException|IllegalArgumentException|InvocationTargetException<a name="line.881"></a> -<span class="sourceLineNo">882</span> exception) {<a name="line.882"></a> -<span class="sourceLineNo">883</span> LOG.warn("Your Hadoop installation's StreamCapabilities implementation doesn't match " +<a name="line.883"></a> -<span class="sourceLineNo">884</span> "our understanding of how it's supposed to work. Please file a JIRA and include " +<a name="line.884"></a> -<span class="sourceLineNo">885</span> "the following stack trace. In the mean time we're interpreting this behavior " +<a name="line.885"></a> -<span class="sourceLineNo">886</span> "difference as a lack of capability support, which will probably cause a failure.",<a name="line.886"></a> -<span class="sourceLineNo">887</span> exception);<a name="line.887"></a> -<span class="sourceLineNo">888</span> }<a name="line.888"></a> -<span class="sourceLineNo">889</span> }<a name="line.889"></a> -<span class="sourceLineNo">890</span> }<a name="line.890"></a> -<span class="sourceLineNo">891</span> return result;<a name="line.891"></a> -<span class="sourceLineNo">892</span> }<a name="line.892"></a> -<span class="sourceLineNo">893</span><a name="line.893"></a> -<span class="sourceLineNo">894</span> /**<a name="line.894"></a> -<span class="sourceLineNo">895</span> * Helper exception for those cases where the place where we need to check a stream capability<a name="line.895"></a> -<span class="sourceLineNo">896</span> * is not where we have the needed context to explain the impact and mitigation for a lack.<a name="line.896"></a> -<span class="sourceLineNo">897</span> */<a name="line.897"></a> -<span class="sourceLineNo">898</span> public static class StreamLacksCapabilityException extends Exception {<a name="line.898"></a> -<span class="sourceLineNo">899</span> public StreamLacksCapabilityException(String message, Throwable cause) {<a name="line.899"></a> -<span class="sourceLineNo">900</span> super(message, cause);<a name="line.900"></a> -<span class="sourceLineNo">901</span> }<a name="line.901"></a> -<span class="sourceLineNo">902</span> public StreamLacksCapabilityException(String message) {<a name="line.902"></a> -<span class="sourceLineNo">903</span> super(message);<a name="line.903"></a> -<span class="sourceLineNo">904</span> }<a name="line.904"></a> -<span class="sourceLineNo">905</span> }<a name="line.905"></a> -<span class="sourceLineNo">906</span><a name="line.906"></a> -<span class="sourceLineNo">907</span>}<a name="line.907"></a> +<span class="sourceLineNo">880</span> Object builder;<a name="line.880"></a> +<span class="sourceLineNo">881</span><a name="line.881"></a> +<span class="sourceLineNo">882</span> builder = createMethod.invoke(fs, path);<a name="line.882"></a> +<span class="sourceLineNo">883</span> builder = overwriteMethod.invoke(builder, overwritable);<a name="line.883"></a> +<span class="sourceLineNo">884</span> builder = bufferSizeMethod.invoke(builder, bufferSize);<a name="line.884"></a> +<span class="sourceLineNo">885</span> builder = blockSizeMethod.invoke(builder, blockSize);<a name="line.885"></a> +<span class="sourceLineNo">886</span> if (isRecursive) {<a name="line.886"></a> +<span class="sourceLineNo">887</span> builder = recursiveMethod.invoke(builder);<a name="line.887"></a> +<span class="sourceLineNo">888</span> }<a name="line.888"></a> +<span class="sourceLineNo">889</span> builder = replicateMethod.invoke(builder);<a name="line.889"></a> +<span class="sourceLineNo">890</span> builder = replicationMethod.invoke(builder, replication);<a name="line.890"></a> +<span class="sourceLineNo">891</span> return (FSDataOutputStream) buildMethod.invoke(builder);<a name="line.891"></a> +<span class="sourceLineNo">892</span> } catch (IllegalAccessException | InvocationTargetException e) {<a name="line.892"></a> +<span class="sourceLineNo">893</span> // Should have caught this failure during initialization, so log full trace here<a name="line.893"></a> +<span class="sourceLineNo">894</span> LOG.warn("Couldn't use reflection with builder API", e);<a name="line.894"></a> +<span class="sourceLineNo">895</span> }<a name="line.895"></a> +<span class="sourceLineNo">896</span> }<a name="line.896"></a> +<span class="sourceLineNo">897</span><a name="line.897"></a> +<span class="sourceLineNo">898</span> if (isRecursive) {<a name="line.898"></a> +<span class="sourceLineNo">899</span> return fs.create(path, overwritable, bufferSize, replication, blockSize, null);<a name="line.899"></a> +<span class="sourceLineNo">900</span> }<a name="line.900"></a> +<span class="sourceLineNo">901</span> return fs.createNonRecursive(path, overwritable, bufferSize, replication, blockSize, null);<a name="line.901"></a> +<span class="sourceLineNo">902</span> }<a name="line.902"></a> +<span class="sourceLineNo">903</span><a name="line.903"></a> +<span class="sourceLineNo">904</span> /**<a name="line.904"></a> +<span class="sourceLineNo">905</span> * Attempt to use builder API via reflection to create a file with the given parameters and<a name="line.905"></a> +<span class="sourceLineNo">906</span> * replication enabled.<a name="line.906"></a> +<span class="sourceLineNo">907</span> */<a name="line.907"></a> +<span class="sourceLineNo">908</span> static FSDataOutputStream createHelper(FileSystem fs, Path path, boolean overwritable)<a name="line.908"></a> +<span class="sourceLineNo">909</span> throws IOException {<a name="line.909"></a> +<span class="sourceLineNo">910</span> if (allMethodsPresent && dfsClass.isInstance(fs)) {<a name="line.910"></a> +<span class="sourceLineNo">911</span> try {<a name="line.911"></a> +<span class="sourceLineNo">912</span> Object builder;<a name="line.912"></a> +<span class="sourceLineNo">913</span><a name="line.913"></a> +<span class="sourceLineNo">914</span> builder = createMethod.invoke(fs, path);<a name="line.914"></a> +<span class="sourceLineNo">915</span> builder = overwriteMethod.invoke(builder, overwritable);<a name="line.915"></a> +<span class="sourceLineNo">916</span> builder = replicateMethod.invoke(builder);<a name="line.916"></a> +<span class="sourceLineNo">917</span> return (FSDataOutputStream) buildMethod.invoke(builder);<a name="line.917"></a> +<span class="sourceLineNo">918</span> } catch (IllegalAccessException | InvocationTargetException e) {<a name="line.918"></a> +<span class="sourceLineNo">919</span> // Should have caught this failure during initialization, so log full trace here<a name="line.919"></a> +<span class="sourceLineNo">920</span> LOG.warn("Couldn't use reflection with builder API", e);<a name="line.920"></a> +<span class="sourceLineNo">921</span> }<a name="line.921"></a> +<span class="sourceLineNo">922</span> }<a name="line.922"></a> +<span class="sourceLineNo">923</span><a name="line.923"></a> +<span class="sourceLineNo">924</span> return fs.create(path, overwritable);<a name="line.924"></a> +<span class="sourceLineNo">925</span> }<a name="line.925"></a> +<span class="sourceLineNo">926</span> }<a name="line.926"></a> +<span class="sourceLineNo">927</span><a name="line.927"></a> +<span class="sourceLineNo">928</span> /**<a name="line.928"></a> +<span class="sourceLineNo">929</span> * Attempt to use builder API via reflection to create a file with the given parameters and<a name="line.929"></a> +<span class="sourceLineNo">930</span> * replication enabled.<a name="line.930"></a> +<span class="sourceLineNo">931</span> * <p><a name="line.931"></a> +<span class="sourceLineNo">932</span> * Will not attempt to enable replication when passed an HFileSystem.<a name="line.932"></a> +<span class="sourceLineNo">933</span> */<a name="line.933"></a> +<span class="sourceLineNo">934</span> public static FSDataOutputStream createForWal(FileSystem fs, Path path, boolean overwritable)<a name="line.934"></a> +<span class="sourceLineNo">935</span> throws IOException {<a name="line.935"></a> +<span class="sourceLineNo">936</span> return DfsBuilderUtility.createHelper(fs, path, overwritable);<a name="line.936"></a> +<span class="sourceLineNo">937</span> }<a name="line.937"></a> +<span class="sourceLineNo">938</span><a name="line.938"></a> +<span class="sourceLineNo">939</span> /**<a name="line.939"></a> +<span class="sourceLineNo">940</span> * Attempt to use builder API via reflection to create a file with the given parameters and<a name="line.940"></a> +<span class="sourceLineNo">941</span> * replication enabled.<a name="line.941"></a> +<span class="sourceLineNo">942</span> * <p><a name="line.942"></a> +<span class="sourceLineNo">943</span> * Will not attempt to enable replication when passed an HFileSystem.<a name="line.943"></a> +<span class="sourceLineNo">944</span> */<a name="line.944"></a> +<span class="sourceLineNo">945</span> public static FSDataOutputStream createForWal(FileSystem fs, Path path, boolean overwritable,<a name="line.945"></a> +<span class="sourceLineNo">946</span> int bufferSize, short replication, long blockSize, boolean isRecursive) throws IOException {<a name="line.946"></a> +<span class="sourceLineNo">947</span> return DfsBuilderUtility.createHelper(fs, path, overwritable, bufferSize, replication,<a name="line.947"></a> +<span class="sourceLineNo">948</span> blockSize, isRecursive);<a name="line.948"></a> +<span class="sourceLineNo">949</span> }<a name="line.949"></a> +<span class="sourceLineNo">950</span><a name="line.950"></a> +<span class="sourceLineNo">951</span> // Holder singleton idiom. JVM spec ensures this will be run at most once per Classloader, and<a name="line.951"></a> +<span class="sourceLineNo">952</span> // not until we attempt to reference it.<a name="line.952"></a> +<span class="sourceLineNo">953</span> private static class StreamCapabilities {<a name="line.953"></a> +<span class="sourceLineNo">954</span> public static final boolean PRESENT;<a name="line.954"></a> +<span class="sourceLineNo">955</span> public static final Class<?> CLASS;<a name="line.955"></a> +<span class="sourceLineNo">956</span> public static final Method METHOD;<a name="line.956"></a> +<span class="sourceLineNo">957</span> static {<a name="line.957"></a> +<span class="sourceLineNo">958</span> boolean tmp = false;<a name="line.958"></a> +<span class="sourceLineNo">959</span> Class<?> clazz = null;<a name="line.959"></a> +<span class="sourceLineNo">960</span> Method method = null;<a name="line.960"></a> +<span class="sourceLineNo">961</span> try {<a name="line.961"></a> +<span class="sourceLineNo">962</span> clazz = Class.forName("org.apache.hadoop.fs.StreamCapabilities");<a name="line.962"></a> +<span class="sourceLineNo">963</span> method = clazz.getMethod("hasCapability", String.class);<a name="line.963"></a> +<span class="sourceLineNo">964</span> tmp = true;<a name="line.964"></a> +<span class="sourceLineNo">965</span> } catch(ClassNotFoundException|NoSuchMethodException|SecurityException exception) {<a name="line.965"></a> +<span class="sourceLineNo">966</span> LOG.warn("Your Hadoop installation does not include the StreamCapabilities class from " +<a name="line.966"></a> +<span class="sourceLineNo">967</span> "HDFS-11644, so we will skip checking if any FSDataOutputStreams actually " +<a name="line.967"></a> +<span class="sourceLineNo">968</span> "support hflush/hsync. If you are running on top of HDFS this probably just " +<a name="line.968"></a> +<span class="sourceLineNo">969</span> "means you have an older version and this can be ignored. If you are running on " +<a name="line.969"></a> +<span class="sourceLineNo">970</span> "top of an alternate FileSystem implementation you should manually verify that " +<a name="line.970"></a> +<span class="sourceLineNo">971</span> "hflush and hsync are implemented; otherwise you risk data loss and hard to " +<a name="line.971"></a> +<span class="sourceLineNo">972</span> "diagnose errors when our assumptions are violated.");<a name="line.972"></a> +<span class="sourceLineNo">973</span> LOG.debug("The first request to check for StreamCapabilities came from this stacktrace.",<a name="line.973"></a> +<span class="sourceLineNo">974</span> exception);<a name="line.974"></a> +<span class="sourceLineNo">975</span> } finally {<a name="line.975"></a> +<span class="sourceLineNo">976</span> PRESENT = tmp;<a name="line.976"></a> +<span class="sourceLineNo">977</span> CLASS = clazz;<a name="line.977"></a> +<span class="sourceLineNo">978</span> METHOD = method;<a name="line.978"></a> +<span class="sourceLineNo">979</span> }<a name="line.979"></a> +<span class="sourceLineNo">980</span> }<a name="line.980"></a> +<span class="sourceLineNo">981</span> }<a name="line.981"></a> +<span class="sourceLineNo">982</span><a name="line.982"></a> +<span class="sourceLineNo">983</span> /**<a name="line.983"></a> +<span class="sourceLineNo">984</span> * If our FileSystem version includes the StreamCapabilities class, check if<a name="line.984"></a> +<span class="sourceLineNo">985</span> * the given stream has a particular capability.<a name="line.985"></a> +<span class="sourceLineNo">986</span> * @param stream capabilities are per-stream instance, so check this one specifically. must not be<a name="line.986"></a> +<span class="sourceLineNo">987</span> * null<a name="line.987"></a> +<span class="sourceLineNo">988</span> * @param capability what to look for, per Hadoop Common's FileSystem docs<a name="line.988"></a> +<span class="sourceLineNo">989</span> * @return true if there are no StreamCapabilities. false if there are, but this stream doesn't<a name="line.989"></a> +<span class="sourceLineNo">990</span> * implement it. return result of asking the stream otherwise.<a name="line.990"></a> +<span class="sourceLineNo">991</span> */<a name="line.991"></a> +<span class="sourceLineNo">992</span> public static boolean hasCapability(FSDataOutputStream stream, String capability) {<a name="line.992"></a> +<span class="sourceLineNo">993</span> // be consistent whether or not StreamCapabilities is present<a name="line.993"></a> +<span class="sourceLineNo">994</span> if (stream == null) {<a name="line.994"></a> +<span class="sourceLineNo">995</span> throw new NullPointerException("stream parameter must not be null.");<a name="line.995"></a> +<span class="sourceLineNo">996</span> }<a name="line.996"></a> +<span class="sourceLineNo">997</span> // If o.a.h.fs.StreamCapabilities doesn't exist, assume everyone does everything<a name="line.997"></a> +<span class="sourceLineNo">998</span> // otherwise old versions of Hadoop will break.<a name="line.998"></a> +<span class="sourceLineNo">999</span> boolean result = true;<a name="line.999"></a> +<span class="sourceLineNo">1000</span> if (StreamCapabilities.PRESENT) {<a name="line.1000"></a> +<span class="sourceLineNo">1001</span> // if StreamCapabilities is present, but the stream doesn't implement it<a name="line.1001"></a> +<span class="sourceLineNo">1002</span> // or we run into a problem invoking the method,<a name="line.1002"></a> +<span class="sourceLineNo">1003</span> // we treat that as equivalent to not declaring anything<a name="line.1003"></a> +<span class="sourceLineNo">1004</span> result = false;<a name="line.1004"></a> +<span class="sourceLineNo">1005</span> if (StreamCapabilities.CLASS.isAssignableFrom(stream.getClass())) {<a name="line.1005"></a> +<span class="sourceLineNo">1006</span> try {<a name="line.1006"></a> +<span class="sourceLineNo">1007</span> result = ((Boolean)StreamCapabilities.METHOD.invoke(stream, capability)).booleanValue();<a name="line.1007"></a> +<span class="sourceLineNo">1008</span> } catch (IllegalAccessException|IllegalArgumentException|InvocationTargetException<a name="line.1008"></a> +<span class="sourceLineNo">1009</span> exception) {<a name="line.1009"></a> +<span class="sourceLineNo">1010</span> LOG.warn("Your Hadoop installation's StreamCapabilities implementation doesn't match " +<a name="line.1010"></a> +<span class="sourceLineNo">1011</span> "our understanding of how it's supposed to work. Please file a JIRA and include " +<a name="line.1011"></a> +<span class="sourceLineNo">1012</span> "the following stack trace. In the mean time we're interpreting this behavior " +<a name="line.1012"></a> +<span class="sourceLineNo">1013</span> "difference as a lack of capability support, which will probably cause a failure.",<a name="line.1013"></a> +<span class="sourceLineNo">1014</span> exception);<a name="line.1014"></a> +<span class="sourceLineNo">1015</span> }<a name="line.1015"></a> +<span class="sourceLineNo">1016</span> }<a name="line.1016"></a> +<span class="sourceLineNo">1017</span> }<a name="line.1017"></a> +<span class="sourceLineNo">1018</span> return result;<a name="line.1018"></a> +<span class="sourceLineNo">1019</span> }<a name="line.1019"></a> +<span class="sourceLineNo">1020</span><a name="line.1020"></a> +<span class="sourceLineNo">1021</span> /**<a name="line.1021"></a> +<span class="sourceLineNo">1022</span> * Helper exception for those cases where the place where we need to check a stream capability<a name="line.1022"></a> +<span class="sourceLineNo">1023</span> * is not where we have the needed context to explain the impact and mitigation for a lack.<a name="line.1023"></a> +<span class="sourceLineNo">1024</span> */<a name="line.1024"></a> +<span class="sourceLineNo">1025</span> public static class StreamLacksCapabilityException extends Exception {<a name="line.1025"></a> +<span class="sourceLineNo">1026</span> public StreamLacksCapabilityException(String message, Throwable cause) {<a name="line.1026"></a> +<span class="sourceLineNo">1027</span> super(message, cause);<a name="line.1027"></a> +<span class="sourceLineNo">1028</span> }<a name="line.1028"></a> +<span class="sourceLineNo">1029</span> public StreamLacksCapabilityException(String message) {<a name="line.1029"></a> +<span class="sourceLineNo">1030</span> super(message);<a name="line.1030"></a> +<span class="sourceLineNo">1031</span> }<a name="line.1031"></a> +<span class="sourceLineNo">1032</span> }<a name="line.1032"></a> +<span class="sourceLineNo">1033</span><a name="line.1033"></a> +<span class="sourceLineNo">1034</span>}<a name="line.1034"></a>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/e10887fe/devapidocs/src-html/org/apache/hadoop/hbase/util/CommonFSUtils.StreamLacksCapabilityException.html ---------------------------------------------------------------------- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/util/CommonFSUtils.StreamLacksCapabilityException.html b/devapidocs/src-html/org/apache/hadoop/hbase/util/CommonFSUtils.StreamLacksCapabilityException.html index c6bfdca..cae3d07 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/util/CommonFSUtils.StreamLacksCapabilityException.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/util/CommonFSUtils.StreamLacksCapabilityException.html @@ -829,90 +829,217 @@ <span class="sourceLineNo">821</span> conf.setIfUnset(dfsKey, Integer.toString(hbaseSize));<a name="line.821"></a> <span class="sourceLineNo">822</span> }<a name="line.822"></a> <span class="sourceLineNo">823</span><a name="line.823"></a> -<span class="sourceLineNo">824</span> // Holder singleton idiom. JVM spec ensures this will be run at most once per Classloader, and<a name="line.824"></a> -<span class="sourceLineNo">825</span> // not until we attempt to reference it.<a name="line.825"></a> -<span class="sourceLineNo">826</span> private static class StreamCapabilities {<a name="line.826"></a> -<span class="sourceLineNo">827</span> public static final boolean PRESENT;<a name="line.827"></a> -<span class="sourceLineNo">828</span> public static final Class<?> CLASS;<a name="line.828"></a> -<span class="sourceLineNo">829</span> public static final Method METHOD;<a name="line.829"></a> -<span class="sourceLineNo">830</span> static {<a name="line.830"></a> -<span class="sourceLineNo">831</span> boolean tmp = false;<a name="line.831"></a> -<span class="sourceLineNo">832</span> Class<?> clazz = null;<a name="line.832"></a> -<span class="sourceLineNo">833</span> Method method = null;<a name="line.833"></a> -<span class="sourceLineNo">834</span> try {<a name="line.834"></a> -<span class="sourceLineNo">835</span> clazz = Class.forName("org.apache.hadoop.fs.StreamCapabilities");<a name="line.835"></a> -<span class="sourceLineNo">836</span> method = clazz.getMethod("hasCapability", String.class);<a name="line.836"></a> -<span class="sourceLineNo">837</span> tmp = true;<a name="line.837"></a> -<span class="sourceLineNo">838</span> } catch(ClassNotFoundException|NoSuchMethodException|SecurityException exception) {<a name="line.838"></a> -<span class="sourceLineNo">839</span> LOG.warn("Your Hadoop installation does not include the StreamCapabilities class from " +<a name="line.839"></a> -<span class="sourceLineNo">840</span> "HDFS-11644, so we will skip checking if any FSDataOutputStreams actually " +<a name="line.840"></a> -<span class="sourceLineNo">841</span> "support hflush/hsync. If you are running on top of HDFS this probably just " +<a name="line.841"></a> -<span class="sourceLineNo">842</span> "means you have an older version and this can be ignored. If you are running on " +<a name="line.842"></a> -<span class="sourceLineNo">843</span> "top of an alternate FileSystem implementation you should manually verify that " +<a name="line.843"></a> -<span class="sourceLineNo">844</span> "hflush and hsync are implemented; otherwise you risk data loss and hard to " +<a name="line.844"></a> -<span class="sourceLineNo">845</span> "diagnose errors when our assumptions are violated.");<a name="line.845"></a> -<span class="sourceLineNo">846</span> LOG.debug("The first request to check for StreamCapabilities came from this stacktrace.",<a name="line.846"></a> -<span class="sourceLineNo">847</span> exception);<a name="line.847"></a> -<span class="sourceLineNo">848</span> } finally {<a name="line.848"></a> -<span class="sourceLineNo">849</span> PRESENT = tmp;<a name="line.849"></a> -<span class="sourceLineNo">850</span> CLASS = clazz;<a name="line.850"></a> -<span class="sourceLineNo">851</span> METHOD = method;<a name="line.851"></a> -<span class="sourceLineNo">852</span> }<a name="line.852"></a> -<span class="sourceLineNo">853</span> }<a name="line.853"></a> -<span class="sourceLineNo">854</span> }<a name="line.854"></a> -<span class="sourceLineNo">855</span><a name="line.855"></a> -<span class="sourceLineNo">856</span> /**<a name="line.856"></a> -<span class="sourceLineNo">857</span> * If our FileSystem version includes the StreamCapabilities class, check if<a name="line.857"></a> -<span class="sourceLineNo">858</span> * the given stream has a particular capability.<a name="line.858"></a> -<span class="sourceLineNo">859</span> * @param stream capabilities are per-stream instance, so check this one specifically. must not be<a name="line.859"></a> -<span class="sourceLineNo">860</span> * null<a name="line.860"></a> -<span class="sourceLineNo">861</span> * @param capability what to look for, per Hadoop Common's FileSystem docs<a name="line.861"></a> -<span class="sourceLineNo">862</span> * @return true if there are no StreamCapabilities. false if there are, but this stream doesn't<a name="line.862"></a> -<span class="sourceLineNo">863</span> * implement it. return result of asking the stream otherwise.<a name="line.863"></a> -<span class="sourceLineNo">864</span> */<a name="line.864"></a> -<span class="sourceLineNo">865</span> public static boolean hasCapability(FSDataOutputStream stream, String capability) {<a name="line.865"></a> -<span class="sourceLineNo">866</span> // be consistent whether or not StreamCapabilities is present<a name="line.866"></a> -<span class="sourceLineNo">867</span> if (stream == null) {<a name="line.867"></a> -<span class="sourceLineNo">868</span> throw new NullPointerException("stream parameter must not be null.");<a name="line.868"></a> -<span class="sourceLineNo">869</span> }<a name="line.869"></a> -<span class="sourceLineNo">870</span> // If o.a.h.fs.StreamCapabilities doesn't exist, assume everyone does everything<a name="line.870"></a> -<span class="sourceLineNo">871</span> // otherwise old versions of Hadoop will break.<a name="line.871"></a> -<span class="sourceLineNo">872</span> boolean result = true;<a name="line.872"></a> -<span class="sourceLineNo">873</span> if (StreamCapabilities.PRESENT) {<a name="line.873"></a> -<span class="sourceLineNo">874</span> // if StreamCapabilities is present, but the stream doesn't implement it<a name="line.874"></a> -<span class="sourceLineNo">875</span> // or we run into a problem invoking the method,<a name="line.875"></a> -<span class="sourceLineNo">876</span> // we treat that as equivalent to not declaring anything<a name="line.876"></a> -<span class="sourceLineNo">877</span> result = false;<a name="line.877"></a> -<span class="sourceLineNo">878</span> if (StreamCapabilities.CLASS.isAssignableFrom(stream.getClass())) {<a name="line.878"></a> +<span class="sourceLineNo">824</span> private static class DfsBuilderUtility {<a name="line.824"></a> +<span class="sourceLineNo">825</span> static Class<?> dfsClass = null;<a name="line.825"></a> +<span class="sourceLineNo">826</span> static Method createMethod;<a name="line.826"></a> +<span class="sourceLineNo">827</span> static Method overwriteMethod;<a name="line.827"></a> +<span class="sourceLineNo">828</span> static Method bufferSizeMethod;<a name="line.828"></a> +<span class="sourceLineNo">829</span> static Method blockSizeMethod;<a name="line.829"></a> +<span class="sourceLineNo">830</span> static Method recursiveMethod;<a name="line.830"></a> +<span class="sourceLineNo">831</span> static Method replicateMethod;<a name="line.831"></a> +<span class="sourceLineNo">832</span> static Method replicationMethod;<a name="line.832"></a> +<span class="sourceLineNo">833</span> static Method buildMethod;<a name="line.833"></a> +<span class="sourceLineNo">834</span> static boolean allMethodsPresent = false;<a name="line.834"></a> +<span class="sourceLineNo">835</span><a name="line.835"></a> +<span class="sourceLineNo">836</span> static {<a name="line.836"></a> +<span class="sourceLineNo">837</span> String dfsName = "org.apache.hadoop.hdfs.DistributedFileSystem";<a name="line.837"></a> +<span class="sourceLineNo">838</span> String builderName = dfsName + "$HdfsDataOutputStreamBuilder";<a name="line.838"></a> +<span class="sourceLineNo">839</span> Class<?> builderClass = null;<a name="line.839"></a> +<span class="sourceLineNo">840</span><a name="line.840"></a> +<span class="sourceLineNo">841</span> try {<a name="line.841"></a> +<span class="sourceLineNo">842</span> dfsClass = Class.forName(dfsName);<a name="line.842"></a> +<span class="sourceLineNo">843</span> } catch (ClassNotFoundException e) {<a name="line.843"></a> +<span class="sourceLineNo">844</span> LOG.debug("{} not available, will not use builder API for file creation.", dfsName);<a name="line.844"></a> +<span class="sourceLineNo">845</span> }<a name="line.845"></a> +<span class="sourceLineNo">846</span> try {<a name="line.846"></a> +<span class="sourceLineNo">847</span> builderClass = Class.forName(builderName);<a name="line.847"></a> +<span class="sourceLineNo">848</span> } catch (ClassNotFoundException e) {<a name="line.848"></a> +<span class="sourceLineNo">849</span> LOG.debug("{} not available, will not use builder API for file creation.", builderName);<a name="line.849"></a> +<span class="sourceLineNo">850</span> }<a name="line.850"></a> +<span class="sourceLineNo">851</span><a name="line.851"></a> +<span class="sourceLineNo">852</span> if (dfsClass != null && builderClass != null) {<a name="line.852"></a> +<span class="sourceLineNo">853</span> try {<a name="line.853"></a> +<span class="sourceLineNo">854</span> createMethod = dfsClass.getMethod("createFile", Path.class);<a name="line.854"></a> +<span class="sourceLineNo">855</span> overwriteMethod = builderClass.getMethod("overwrite", boolean.class);<a name="line.855"></a> +<span class="sourceLineNo">856</span> bufferSizeMethod = builderClass.getMethod("bufferSize", int.class);<a name="line.856"></a> +<span class="sourceLineNo">857</span> blockSizeMethod = builderClass.getMethod("blockSize", long.class);<a name="line.857"></a> +<span class="sourceLineNo">858</span> recursiveMethod = builderClass.getMethod("recursive");<a name="line.858"></a> +<span class="sourceLineNo">859</span> replicateMethod = builderClass.getMethod("replicate");<a name="line.859"></a> +<span class="sourceLineNo">860</span> replicationMethod = builderClass.getMethod("replication", short.class);<a name="line.860"></a> +<span class="sourceLineNo">861</span> buildMethod = builderClass.getMethod("build");<a name="line.861"></a> +<span class="sourceLineNo">862</span><a name="line.862"></a> +<span class="sourceLineNo">863</span> allMethodsPresent = true;<a name="line.863"></a> +<span class="sourceLineNo">864</span> LOG.debug("Using builder API via reflection for DFS file creation.");<a name="line.864"></a> +<span class="sourceLineNo">865</span> } catch (NoSuchMethodException e) {<a name="line.865"></a> +<span class="sourceLineNo">866</span> LOG.debug("Could not find method on builder; will use old DFS API for file creation {}",<a name="line.866"></a> +<span class="sourceLineNo">867</span> e.getMessage());<a name="line.867"></a> +<span class="sourceLineNo">868</span> }<a name="line.868"></a> +<span class="sourceLineNo">869</span> }<a name="line.869"></a> +<span class="sourceLineNo">870</span> }<a name="line.870"></a> +<span class="sourceLineNo">871</span><a name="line.871"></a> +<span class="sourceLineNo">872</span> /**<a name="line.872"></a> +<span class="sourceLineNo">873</span> * Attempt to use builder API via reflection to create a file with the given parameters and<a name="line.873"></a> +<span class="sourceLineNo">874</span> * replication enabled.<a name="line.874"></a> +<span class="sourceLineNo">875</span> */<a name="line.875"></a> +<span class="sourceLineNo">876</span> static FSDataOutputStream createHelper(FileSystem fs, Path path, boolean overwritable,<a name="line.876"></a> +<span class="sourceLineNo">877</span> int bufferSize, short replication, long blockSize, boolean isRecursive) throws IOException {<a name="line.877"></a> +<span class="sourceLineNo">878</span> if (allMethodsPresent && dfsClass.isInstance(fs)) {<a name="line.878"></a> <span class="sourceLineNo">879</span> try {<a name="line.879"></a> -<span class="sourceLineNo">880</span> result = ((Boolean)StreamCapabilities.METHOD.invoke(stream, capability)).booleanValue();<a name="line.880"></a> -<span class="sourceLineNo">881</span> } catch (IllegalAccessException|IllegalArgumentException|InvocationTargetException<a name="line.881"></a> -<span class="sourceLineNo">882</span> exception) {<a name="line.882"></a> -<span class="sourceLineNo">883</span> LOG.warn("Your Hadoop installation's StreamCapabilities implementation doesn't match " +<a name="line.883"></a> -<span class="sourceLineNo">884</span> "our understanding of how it's supposed to work. Please file a JIRA and include " +<a name="line.884"></a> -<span class="sourceLineNo">885</span> "the following stack trace. In the mean time we're interpreting this behavior " +<a name="line.885"></a> -<span class="sourceLineNo">886</span> "difference as a lack of capability support, which will probably cause a failure.",<a name="line.886"></a> -<span class="sourceLineNo">887</span> exception);<a name="line.887"></a> -<span class="sourceLineNo">888</span> }<a name="line.888"></a> -<span class="sourceLineNo">889</span> }<a name="line.889"></a> -<span class="sourceLineNo">890</span> }<a name="line.890"></a> -<span class="sourceLineNo">891</span> return result;<a name="line.891"></a> -<span class="sourceLineNo">892</span> }<a name="line.892"></a> -<span class="sourceLineNo">893</span><a name="line.893"></a> -<span class="sourceLineNo">894</span> /**<a name="line.894"></a> -<span class="sourceLineNo">895</span> * Helper exception for those cases where the place where we need to check a stream capability<a name="line.895"></a> -<span class="sourceLineNo">896</span> * is not where we have the needed context to explain the impact and mitigation for a lack.<a name="line.896"></a> -<span class="sourceLineNo">897</span> */<a name="line.897"></a> -<span class="sourceLineNo">898</span> public static class StreamLacksCapabilityException extends Exception {<a name="line.898"></a> -<span class="sourceLineNo">899</span> public StreamLacksCapabilityException(String message, Throwable cause) {<a name="line.899"></a> -<span class="sourceLineNo">900</span> super(message, cause);<a name="line.900"></a> -<span class="sourceLineNo">901</span> }<a name="line.901"></a> -<span class="sourceLineNo">902</span> public StreamLacksCapabilityException(String message) {<a name="line.902"></a> -<span class="sourceLineNo">903</span> super(message);<a name="line.903"></a> -<span class="sourceLineNo">904</span> }<a name="line.904"></a> -<span class="sourceLineNo">905</span> }<a name="line.905"></a> -<span class="sourceLineNo">906</span><a name="line.906"></a> -<span class="sourceLineNo">907</span>}<a name="line.907"></a> +<span class="sourceLineNo">880</span> Object builder;<a name="line.880"></a> +<span class="sourceLineNo">881</span><a name="line.881"></a> +<span class="sourceLineNo">882</span> builder = createMethod.invoke(fs, path);<a name="line.882"></a> +<span class="sourceLineNo">883</span> builder = overwriteMethod.invoke(builder, overwritable);<a name="line.883"></a> +<span class="sourceLineNo">884</span> builder = bufferSizeMethod.invoke(builder, bufferSize);<a name="line.884"></a> +<span class="sourceLineNo">885</span> builder = blockSizeMethod.invoke(builder, blockSize);<a name="line.885"></a> +<span class="sourceLineNo">886</span> if (isRecursive) {<a name="line.886"></a> +<span class="sourceLineNo">887</span> builder = recursiveMethod.invoke(builder);<a name="line.887"></a> +<span class="sourceLineNo">888</span> }<a name="line.888"></a> +<span class="sourceLineNo">889</span> builder = replicateMethod.invoke(builder);<a name="line.889"></a> +<span class="sourceLineNo">890</span> builder = replicationMethod.invoke(builder, replication);<a name="line.890"></a> +<span class="sourceLineNo">891</span> return (FSDataOutputStream) buildMethod.invoke(builder);<a name="line.891"></a> +<span class="sourceLineNo">892</span> } catch (IllegalAccessException | InvocationTargetException e) {<a name="line.892"></a> +<span class="sourceLineNo">893</span> // Should have caught this failure during initialization, so log full trace here<a name="line.893"></a> +<span class="sourceLineNo">894</span> LOG.warn("Couldn't use reflection with builder API", e);<a name="line.894"></a> +<span class="sourceLineNo">895</span> }<a name="line.895"></a> +<span class="sourceLineNo">896</span> }<a name="line.896"></a> +<span class="sourceLineNo">897</span><a name="line.897"></a> +<span class="sourceLineNo">898</span> if (isRecursive) {<a name="line.898"></a> +<span class="sourceLineNo">899</span> return fs.create(path, overwritable, bufferSize, replication, blockSize, null);<a name="line.899"></a> +<span class="sourceLineNo">900</span> }<a name="line.900"></a> +<span class="sourceLineNo">901</span> return fs.createNonRecursive(path, overwritable, bufferSize, replication, blockSize, null);<a name="line.901"></a> +<span class="sourceLineNo">902</span> }<a name="line.902"></a> +<span class="sourceLineNo">903</span><a name="line.903"></a> +<span class="sourceLineNo">904</span> /**<a name="line.904"></a> +<span class="sourceLineNo">905</span> * Attempt to use builder API via reflection to create a file with the given parameters and<a name="line.905"></a> +<span class="sourceLineNo">906</span> * replication enabled.<a name="line.906"></a> +<span class="sourceLineNo">907</span> */<a name="line.907"></a> +<span class="sourceLineNo">908</span> static FSDataOutputStream createHelper(FileSystem fs, Path path, boolean overwritable)<a name="line.908"></a> +<span class="sourceLineNo">909</span> throws IOException {<a name="line.909"></a> +<span class="sourceLineNo">910</span> if (allMethodsPresent && dfsClass.isInstance(fs)) {<a name="line.910"></a> +<span class="sourceLineNo">911</span> try {<a name="line.911"></a> +<span class="sourceLineNo">912</span> Object builder;<a name="line.912"></a> +<span class="sourceLineNo">913</span><a name="line.913"></a> +<span class="sourceLineNo">914</span> builder = createMethod.invoke(fs, path);<a name="line.914"></a> +<span class="sourceLineNo">915</span> builder = overwriteMethod.invoke(builder, overwritable);<a name="line.915"></a> +<span class="sourceLineNo">916</span> builder = replicateMethod.invoke(builder);<a name="line.916"></a> +<span class="sourceLineNo">917</span> return (FSDataOutputStream) buildMethod.invoke(builder);<a name="line.917"></a> +<span class="sourceLineNo">918</span> } catch (IllegalAccessException | InvocationTargetException e) {<a name="line.918"></a> +<span class="sourceLineNo">919</span> // Should have caught this failure during initialization, so log full trace here<a name="line.919"></a> +<span class="sourceLineNo">920</span> LOG.warn("Couldn't use reflection with builder API", e);<a name="line.920"></a> +<span class="sourceLineNo">921</span> }<a name="line.921"></a> +<span class="sourceLineNo">922</span> }<a name="line.922"></a> +<span class="sourceLineNo">923</span><a name="line.923"></a> +<span class="sourceLineNo">924</span> return fs.create(path, overwritable);<a name="line.924"></a> +<span class="sourceLineNo">925</span> }<a name="line.925"></a> +<span class="sourceLineNo">926</span> }<a name="line.926"></a> +<span class="sourceLineNo">927</span><a name="line.927"></a> +<span class="sourceLineNo">928</span> /**<a name="line.928"></a> +<span class="sourceLineNo">929</span> * Attempt to use builder API via reflection to create a file with the given parameters and<a name="line.929"></a> +<span class="sourceLineNo">930</span> * replication enabled.<a name="line.930"></a> +<span class="sourceLineNo">931</span> * <p><a name="line.931"></a> +<span class="sourceLineNo">932</span> * Will not attempt to enable replication when passed an HFileSystem.<a name="line.932"></a> +<span class="sourceLineNo">933</span> */<a name="line.933"></a> +<span class="sourceLineNo">934</span> public static FSDataOutputStream createForWal(FileSystem fs, Path path, boolean overwritable)<a name="line.934"></a> +<span class="sourceLineNo">935</span> throws IOException {<a name="line.935"></a> +<span class="sourceLineNo">936</span> return DfsBuilderUtility.createHelper(fs, path, overwritable);<a name="line.936"></a> +<span class="sourceLineNo">937</span> }<a name="line.937"></a> +<span class="sourceLineNo">938</span><a name="line.938"></a> +<span class="sourceLineNo">939</span> /**<a name="line.939"></a> +<span class="sourceLineNo">940</span> * Attempt to use builder API via reflection to create a file with the given parameters and<a name="line.940"></a> +<span class="sourceLineNo">941</span> * replication enabled.<a name="line.941"></a> +<span class="sourceLineNo">942</span> * <p><a name="line.942"></a> +<span class="sourceLineNo">943</span> * Will not attempt to enable replication when passed an HFileSystem.<a name="line.943"></a> +<span class="sourceLineNo">944</span> */<a name="line.944"></a> +<span class="sourceLineNo">945</span> public static FSDataOutputStream createForWal(FileSystem fs, Path path, boolean overwritable,<a name="line.945"></a> +<span class="sourceLineNo">946</span> int bufferSize, short replication, long blockSize, boolean isRecursive) throws IOException {<a name="line.946"></a> +<span class="sourceLineNo">947</span> return DfsBuilderUtility.createHelper(fs, path, overwritable, bufferSize, replication,<a name="line.947"></a> +<span class="sourceLineNo">948</span> blockSize, isRecursive);<a name="line.948"></a> +<span class="sourceLineNo">949</span> }<a name="line.949"></a> +<span class="sourceLineNo">950</span><a name="line.950"></a> +<span class="sourceLineNo">951</span> // Holder singleton idiom. JVM spec ensures this will be run at most once per Classloader, and<a name="line.951"></a> +<span class="sourceLineNo">952</span> // not until we attempt to reference it.<a name="line.952"></a> +<span class="sourceLineNo">953</span> private static class StreamCapabilities {<a name="line.953"></a> +<span class="sourceLineNo">954</span> public static final boolean PRESENT;<a name="line.954"></a> +<span class="sourceLineNo">955</span> public static final Class<?> CLASS;<a name="line.955"></a> +<span class="sourceLineNo">956</span> public static final Method METHOD;<a name="line.956"></a> +<span class="sourceLineNo">957</span> static {<a name="line.957"></a> +<span class="sourceLineNo">958</span> boolean tmp = false;<a name="line.958"></a> +<span class="sourceLineNo">959</span> Class<?> clazz = null;<a name="line.959"></a> +<span class="sourceLineNo">960</span> Method method = null;<a name="line.960"></a> +<span class="sourceLineNo">961</span> try {<a name="line.961"></a> +<span class="sourceLineNo">962</span> clazz = Class.forName("org.apache.hadoop.fs.StreamCapabilities");<a name="line.962"></a> +<span class="sourceLineNo">963</span> method = clazz.getMethod("hasCapability", String.class);<a name="line.963"></a> +<span class="sourceLineNo">964</span> tmp = true;<a name="line.964"></a> +<span class="sourceLineNo">965</span> } catch(ClassNotFoundException|NoSuchMethodException|SecurityException exception) {<a name="line.965"></a> +<span class="sourceLineNo">966</span> LOG.warn("Your Hadoop installation does not include the StreamCapabilities class from " +<a name="line.966"></a> +<span class="sourceLineNo">967</span> "HDFS-11644, so we will skip checking if any FSDataOutputStreams actually " +<a name="line.967"></a> +<span class="sourceLineNo">968</span> "support hflush/hsync. If you are running on top of HDFS this probably just " +<a name="line.968"></a> +<span class="sourceLineNo">969</span> "means you have an older version and this can be ignored. If you are running on " +<a name="line.969"></a> +<span class="sourceLineNo">970</span> "top of an alternate FileSystem implementation you should manually verify that " +<a name="line.970"></a> +<span class="sourceLineNo">971</span> "hflush and hsync are implemented; otherwise you risk data loss and hard to " +<a name="line.971"></a> +<span class="sourceLineNo">972</span> "diagnose errors when our assumptions are violated.");<a name="line.972"></a> +<span class="sourceLineNo">973</span> LOG.debug("The first request to check for StreamCapabilities came from this stacktrace.",<a name="line.973"></a> +<span class="sourceLineNo">974</span> exception);<a name="line.974"></a> +<span class="sourceLineNo">975</span> } finally {<a name="line.975"></a> +<span class="sourceLineNo">976</span> PRESENT = tmp;<a name="line.976"></a> +<span class="sourceLineNo">977</span> CLASS = clazz;<a name="line.977"></a> +<span class="sourceLineNo">978</span> METHOD = method;<a name="line.978"></a> +<span class="sourceLineNo">979</span> }<a name="line.979"></a> +<span class="sourceLineNo">980</span> }<a name="line.980"></a> +<span class="sourceLineNo">981</span> }<a name="line.981"></a> +<span class="sourceLineNo">982</span><a name="line.982"></a> +<span class="sourceLineNo">983</span> /**<a name="line.983"></a> +<span class="sourceLineNo">984</span> * If our FileSystem version includes the StreamCapabilities class, check if<a name="line.984"></a> +<span class="sourceLineNo">985</span> * the given stream has a particular capability.<a name="line.985"></a> +<span class="sourceLineNo">986</span> * @param stream capabilities are per-stream instance, so check this one specifically. must not be<a name="line.986"></a> +<span class="sourceLineNo">987</span> * null<a name="line.987"></a> +<span class="sourceLineNo">988</span> * @param capability what to look for, per Hadoop Common's FileSystem docs<a name="line.988"></a> +<span class="sourceLineNo">989</span> * @return true if there are no StreamCapabilities. false if there are, but this stream doesn't<a name="line.989"></a> +<span class="sourceLineNo">990</span> * implement it. return result of asking the stream otherwise.<a name="line.990"></a> +<span class="sourceLineNo">991</span> */<a name="line.991"></a> +<span class="sourceLineNo">992</span> public static boolean hasCapability(FSDataOutputStream stream, String capability) {<a name="line.992"></a> +<span class="sourceLineNo">993</span> // be consistent whether or not StreamCapabilities is present<a name="line.993"></a> +<span class="sourceLineNo">994</span> if (stream == null) {<a name="line.994"></a> +<span class="sourceLineNo">995</span> throw new NullPointerException("stream parameter must not be null.");<a name="line.995"></a> +<span class="sourceLineNo">996</span> }<a name="line.996"></a> +<span class="sourceLineNo">997</span> // If o.a.h.fs.StreamCapabilities doesn't exist, assume everyone does everything<a name="line.997"></a> +<span class="sourceLineNo">998</span> // otherwise old versions of Hadoop will break.<a name="line.998"></a> +<span class="sourceLineNo">999</span> boolean result = true;<a name="line.999"></a> +<span class="sourceLineNo">1000</span> if (StreamCapabilities.PRESENT) {<a name="line.1000"></a> +<span class="sourceLineNo">1001</span> // if StreamCapabilities is present, but the stream doesn't implement it<a name="line.1001"></a> +<span class="sourceLineNo">1002</span> // or we run into a problem invoking the method,<a name="line.1002"></a> +<span class="sourceLineNo">1003</span> // we treat that as equivalent to not declaring anything<a name="line.1003"></a> +<span class="sourceLineNo">1004</span> result = false;<a name="line.1004"></a> +<span class="sourceLineNo">1005</span> if (StreamCapabilities.CLASS.isAssignableFrom(stream.getClass())) {<a name="line.1005"></a> +<span class="sourceLineNo">1006</span> try {<a name="line.1006"></a> +<span class="sourceLineNo">1007</span> result = ((Boolean)StreamCapabilities.METHOD.invoke(stream, capability)).booleanValue();<a name="line.1007"></a> +<span class="sourceLineNo">1008</span> } catch (IllegalAccessException|IllegalArgumentException|InvocationTargetException<a name="line.1008"></a> +<span class="sourceLineNo">1009</span> exception) {<a name="line.1009"></a> +<span class="sourceLineNo">1010</span> LOG.warn("Your Hadoop installation's StreamCapabilities implementation doesn't match " +<a name="line.1010"></a> +<span class="sourceLineNo">1011</span> "our understanding of how it's supposed to work. Please file a JIRA and include " +<a name="line.1011"></a> +<span class="sourceLineNo">1012</span> "the following stack trace. In the mean time we're interpreting this behavior " +<a name="line.1012"></a> +<span class="sourceLineNo">1013</span> "difference as a lack of capability support, which will probably cause a failure.",<a name="line.1013"></a> +<span class="sourceLineNo">1014</span> exception);<a name="line.1014"></a> +<span class="sourceLineNo">1015</span> }<a name="line.1015"></a> +<span class="sourceLineNo">1016</span> }<a name="line.1016"></a> +<span class="sourceLineNo">1017</span> }<a name="line.1017"></a> +<span class="sourceLineNo">1018</span> return result;<a name="line.1018"></a> +<span class="sourceLineNo">1019</span> }<a name="line.1019"></a> +<span class="sourceLineNo">1020</span><a name="line.1020"></a> +<span class="sourceLineNo">1021</span> /**<a name="line.1021"></a> +<span class="sourceLineNo">1022</span> * Helper exception for those cases where the place where we need to check a stream capability<a name="line.1022"></a> +<span class="sourceLineNo">1023</span> * is not where we have the needed context to explain the impact and mitigation for a lack.<a name="line.1023"></a> +<span class="sourceLineNo">1024</span> */<a name="line.1024"></a> +<span class="sourceLineNo">1025</span> public static class StreamLacksCapabilityException extends Exception {<a name="line.1025"></a> +<span class="sourceLineNo">1026</span> public StreamLacksCapabilityException(String message, Throwable cause) {<a name="line.1026"></a> +<span class="sourceLineNo">1027</span> super(message, cause);<a name="line.1027"></a> +<span class="sourceLineNo">1028</span> }<a name="line.1028"></a> +<span class="sourceLineNo">1029</span> public StreamLacksCapabilityException(String message) {<a name="line.1029"></a> +<span class="sourceLineNo">1030</span> super(message);<a name="line.1030"></a> +<span class="sourceLineNo">1031</span> }<a name="line.1031"></a> +<span class="sourceLineNo">1032</span> }<a name="line.1032"></a> +<span class="sourceLineNo">1033</span><a name="line.1033"></a> +<span class="sourceLineNo">1034</span>}<a name="line.1034"></a>
