Repository: twill Updated Branches: refs/heads/master f866b50d7 -> dbbc2a349
(TWILL-237) Twill is using hdfs HAUtil api that is nont-compatible with hadoop 2.8 + Use Java's MethodHandle (dynamic lang support) rather than Method (reflection) in FileContextLocationUtil + Expose static API rather than the handle itself This closes #55 on Github Signed-off-by: Terence Yim <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/twill/repo Commit: http://git-wip-us.apache.org/repos/asf/twill/commit/dbbc2a34 Tree: http://git-wip-us.apache.org/repos/asf/twill/tree/dbbc2a34 Diff: http://git-wip-us.apache.org/repos/asf/twill/diff/dbbc2a34 Branch: refs/heads/master Commit: dbbc2a349636cbc2f0c5991f5ad5281d97c0cbb0 Parents: f866b50 Author: Sudheesh Katkam <[email protected]> Authored: Fri Jun 16 15:42:18 2017 -0700 Committer: Terence Yim <[email protected]> Committed: Fri Aug 4 10:22:59 2017 -0700 ---------------------------------------------------------------------- .../twill/filesystem/FileContextLocation.java | 6 +- .../filesystem/FileContextLocationUtil.java | 77 ++++++++++++++++++++ 2 files changed, 81 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/twill/blob/dbbc2a34/twill-yarn/src/main/java/org/apache/twill/filesystem/FileContextLocation.java ---------------------------------------------------------------------- diff --git a/twill-yarn/src/main/java/org/apache/twill/filesystem/FileContextLocation.java b/twill-yarn/src/main/java/org/apache/twill/filesystem/FileContextLocation.java index 8ccd180..f407f01 100644 --- a/twill-yarn/src/main/java/org/apache/twill/filesystem/FileContextLocation.java +++ b/twill-yarn/src/main/java/org/apache/twill/filesystem/FileContextLocation.java @@ -19,6 +19,7 @@ package org.apache.twill.filesystem; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; + import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.FileContext; @@ -28,7 +29,6 @@ import org.apache.hadoop.fs.ParentNotDirectoryException; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.hdfs.HAUtil; import org.apache.hadoop.security.AccessControlException; import java.io.FileNotFoundException; @@ -41,6 +41,7 @@ import java.util.EnumSet; import java.util.List; import java.util.Objects; import java.util.UUID; + import javax.annotation.Nullable; /** @@ -162,7 +163,8 @@ final class FileContextLocation implements Location { // append "port" to the path URI, while the DistributedFileSystem always use the cluster logical // name, which doesn't allow having port in it. URI uri = path.toUri(); - if (HAUtil.isLogicalUri(locationFactory.getConfiguration(), uri)) { + + if (FileContextLocationUtil.useLogicalUri(locationFactory.getConfiguration(), uri)) { try { // Need to strip out the port if in HA return new URI(uri.getScheme(), uri.getUserInfo(), uri.getHost(), http://git-wip-us.apache.org/repos/asf/twill/blob/dbbc2a34/twill-yarn/src/main/java/org/apache/twill/filesystem/FileContextLocationUtil.java ---------------------------------------------------------------------- diff --git a/twill-yarn/src/main/java/org/apache/twill/filesystem/FileContextLocationUtil.java b/twill-yarn/src/main/java/org/apache/twill/filesystem/FileContextLocationUtil.java new file mode 100644 index 0000000..964cfdd --- /dev/null +++ b/twill-yarn/src/main/java/org/apache/twill/filesystem/FileContextLocationUtil.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.twill.filesystem; + +import com.google.common.base.Throwables; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.HAUtil; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.net.URI; + +/** + * Utility class. + */ +final class FileContextLocationUtil { + + // To check whether logical URI is needed, hdfs.HAUtil class is used. But the class is meant for internal purposes, + // and in Hadoop 2.8, the method was renamed from "isLogicalUri" to "useLogicalUri". So resolve to the + // correct method. + private static final MethodHandle HA_UTIL_USE_LOGICAL_URI_HANDLE; + + private static MethodHandle lookupInHAUtil(final String methodName) + throws NoSuchMethodException, IllegalAccessException { + return MethodHandles.publicLookup() + .findStatic(HAUtil.class, methodName, + MethodType.methodType(boolean.class, new Class[]{Configuration.class, URI.class})); + } + + static { + MethodHandle handle; + try { + try { + // hadoop version < 2.8 + handle = lookupInHAUtil("isLogicalUri"); + } catch (NoSuchMethodException ignored) { + try { + // hadoop version = 2.8 + handle = lookupInHAUtil("useLogicalUri"); + } catch (NoSuchMethodException e) { + throw Throwables.propagate(e); + } + } + } catch (IllegalAccessException e) { + throw Throwables.propagate(e); + } + HA_UTIL_USE_LOGICAL_URI_HANDLE = handle; + } + + static boolean useLogicalUri(final Configuration configuration, final URI uri) { + try { + return (Boolean) HA_UTIL_USE_LOGICAL_URI_HANDLE.invoke(configuration, uri); + } catch (Throwable e) { + throw Throwables.propagate(e); + } + } + + private FileContextLocationUtil() { + } +}
