Repository: drill Updated Branches: refs/heads/1.3.0 [created] e465cd90c
DRILL-4065: Maintain a cache of created UserGroupInformation objects so that we don't accidentally generate thousands of FileSystem objects & threads. Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/ba4baea7 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/ba4baea7 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/ba4baea7 Branch: refs/heads/1.3.0 Commit: ba4baea736b24daac3d9b1a8d3ba198e412f6887 Parents: e78e286 Author: Jacques Nadeau <[email protected]> Authored: Tue Nov 10 16:04:09 2015 -0800 Committer: Jacques Nadeau <[email protected]> Committed: Tue Nov 10 16:04:09 2015 -0800 ---------------------------------------------------------------------- .../drill/exec/util/ImpersonationUtil.java | 80 ++++++++++++++++++-- 1 file changed, 72 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/ba4baea7/exec/java-exec/src/main/java/org/apache/drill/exec/util/ImpersonationUtil.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/ImpersonationUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/ImpersonationUtil.java index 37fdc40..38d2797 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/ImpersonationUtil.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/ImpersonationUtil.java @@ -17,18 +17,24 @@ */ package org.apache.drill.exec.util; -import com.google.common.base.Splitter; -import com.google.common.base.Strings; -import com.google.common.collect.Sets; +import java.io.IOException; +import java.security.PrivilegedExceptionAction; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; + import org.apache.drill.common.exceptions.DrillRuntimeException; import org.apache.drill.exec.ops.OperatorStats; import org.apache.drill.exec.store.dfs.DrillFileSystem; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; -import java.io.IOException; -import java.security.PrivilegedExceptionAction; -import java.util.Set; +import com.google.common.base.Splitter; +import com.google.common.base.Strings; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.collect.Sets; /** * Utilities for impersonation purpose. @@ -36,8 +42,66 @@ import java.util.Set; public class ImpersonationUtil { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ImpersonationUtil.class); + private static final LoadingCache<Key, UserGroupInformation> CACHE = CacheBuilder.newBuilder() + .maximumSize(100) + .expireAfterAccess(60, TimeUnit.MINUTES) + .build(new CacheLoader<Key, UserGroupInformation>() { + @Override + public UserGroupInformation load(Key key) throws Exception { + return UserGroupInformation.createProxyUser(key.proxyUserName, key.loginUser); + } + }); + private static final Splitter SPLITTER = Splitter.on(',').trimResults().omitEmptyStrings(); + private static class Key { + final String proxyUserName; + final UserGroupInformation loginUser; + + public Key(String proxyUserName, UserGroupInformation loginUser) { + super(); + this.proxyUserName = proxyUserName; + this.loginUser = loginUser; + } + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((loginUser == null) ? 0 : loginUser.hashCode()); + result = prime * result + ((proxyUserName == null) ? 0 : proxyUserName.hashCode()); + return result; + } + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Key other = (Key) obj; + if (loginUser == null) { + if (other.loginUser != null) { + return false; + } + } else if (!loginUser.equals(other.loginUser)) { + return false; + } + if (proxyUserName == null) { + if (other.proxyUserName != null) { + return false; + } + } else if (!proxyUserName.equals(other.proxyUserName)) { + return false; + } + return true; + } + + + } /** * Create and return proxy user {@link org.apache.hadoop.security.UserGroupInformation} of operator owner if operator * owner is valid. Otherwise create and return proxy user {@link org.apache.hadoop.security.UserGroupInformation} for @@ -83,8 +147,8 @@ public class ImpersonationUtil { return getProcessUserUGI(); } - return UserGroupInformation.createProxyUser(proxyUserName, UserGroupInformation.getLoginUser()); - } catch(IOException e) { + return CACHE.get(new Key(proxyUserName, UserGroupInformation.getLoginUser())); + } catch (IOException | ExecutionException e) { final String errMsg = "Failed to create proxy user UserGroupInformation object: " + e.getMessage(); logger.error(errMsg, e); throw new DrillRuntimeException(errMsg, e);
