[02/50] [abbrv] hadoop git commit: HADOOP-11402. Negative user-to-group cache entries are never cleared for never-again-accessed users. Contributed by Varun Saxena.

2015-01-13 Thread cnauroth 
HADOOP-11402. Negative user-to-group cache entries are never cleared for 
never-again-accessed users. Contributed by Varun Saxena.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/53caeaa1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/53caeaa1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/53caeaa1

Branch: refs/heads/HDFS-6994
Commit: 53caeaa16b1450b54e994c77f5d0c8a767b88d57
Parents: 94d342e
Author: Benoy Antony be...@apache.org
Authored: Mon Jan 5 15:06:46 2015 -0800
Committer: Benoy Antony be...@apache.org
Committed: Mon Jan 5 15:06:46 2015 -0800

--
 .../java/org/apache/hadoop/security/Groups.java | 36 +--
 .../hadoop/security/TestGroupsCaching.java  | 48 
 2 files changed, 71 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/53caeaa1/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
index f3c5094..9fd39b0 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
@@ -23,12 +23,14 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
+import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Ticker;
 import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.Cache;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
 import org.apache.hadoop.HadoopIllegalArgumentException;
@@ -60,14 +62,13 @@ public class Groups {
   private final GroupMappingServiceProvider impl;
 
   private final LoadingCacheString, ListString cache;
-  private final ConcurrentHashMapString, Long negativeCacheMask =
-new ConcurrentHashMapString, Long();
   private final MapString, ListString staticUserToGroupsMap =
   new HashMapString, ListString();
   private final long cacheTimeout;
   private final long negativeCacheTimeout;
   private final long warningDeltaMs;
   private final Timer timer;
+  private SetString negativeCache;
 
   public Groups(Configuration conf) {
 this(conf, new Timer());
@@ -99,11 +100,24 @@ public class Groups {
   .expireAfterWrite(10 * cacheTimeout, TimeUnit.MILLISECONDS)
   .build(new GroupCacheLoader());
 
+if(negativeCacheTimeout  0) {
+  CacheString, Boolean tempMap = CacheBuilder.newBuilder()
+.expireAfterWrite(negativeCacheTimeout, TimeUnit.MILLISECONDS)
+.ticker(new TimerToTickerAdapter(timer))
+.build();
+  negativeCache = Collections.newSetFromMap(tempMap.asMap());
+}
+
 if(LOG.isDebugEnabled())
   LOG.debug(Group mapping impl= + impl.getClass().getName() + 
   ; cacheTimeout= + cacheTimeout + ; warningDeltaMs= +
   warningDeltaMs);
   }
+  
+  @VisibleForTesting
+  SetString getNegativeCache() {
+return negativeCache;
+  }
 
   /*
* Parse the hadoop.user.group.static.mapping.overrides configuration to
@@ -159,13 +173,8 @@ public class Groups {
 
 // Check the negative cache first
 if (isNegativeCacheEnabled()) {
-  Long expirationTime = negativeCacheMask.get(user);
-  if (expirationTime != null) {
-if (timer.monotonicNow()  expirationTime) {
-  throw noGroupsForUser(user);
-} else {
-  negativeCacheMask.remove(user, expirationTime);
-}
+  if (negativeCache.contains(user)) {
+throw noGroupsForUser(user);
   }
 }
 
@@ -212,8 +221,7 @@ public class Groups {
 
   if (groups.isEmpty()) {
 if (isNegativeCacheEnabled()) {
-  long expirationTime = timer.monotonicNow() + negativeCacheTimeout;
-  negativeCacheMask.put(user, expirationTime);
+  negativeCache.add(user);
 }
 
 // We throw here to prevent Cache from retaining an empty group
@@ -252,7 +260,9 @@ public class Groups {
   LOG.warn(Error refreshing groups cache, e);
 }
 cache.invalidateAll();
-negativeCacheMask.clear();
+if(isNegativeCacheEnabled()) {
+  negativeCache.clear();
+}
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/53caeaa1/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java

[03/34] hadoop git commit: HADOOP-11402. Negative user-to-group cache entries are never cleared for never-again-accessed users. Contributed by Varun Saxena.

2015-01-12 Thread zhz 
HADOOP-11402. Negative user-to-group cache entries are never cleared for 
never-again-accessed users. Contributed by Varun Saxena.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9d740be6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9d740be6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9d740be6

Branch: refs/heads/HDFS-EC
Commit: 9d740be670a3ec08a0c4c96c4ba74d02d383b647
Parents: e38cd05
Author: Benoy Antony be...@apache.org
Authored: Mon Jan 5 15:06:46 2015 -0800
Committer: Zhe Zhang z...@apache.org
Committed: Mon Jan 12 10:17:58 2015 -0800

--
 .../java/org/apache/hadoop/security/Groups.java | 36 +--
 .../hadoop/security/TestGroupsCaching.java  | 48 
 2 files changed, 71 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d740be6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
index f3c5094..9fd39b0 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
@@ -23,12 +23,14 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
+import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Ticker;
 import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.Cache;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
 import org.apache.hadoop.HadoopIllegalArgumentException;
@@ -60,14 +62,13 @@ public class Groups {
   private final GroupMappingServiceProvider impl;
 
   private final LoadingCacheString, ListString cache;
-  private final ConcurrentHashMapString, Long negativeCacheMask =
-new ConcurrentHashMapString, Long();
   private final MapString, ListString staticUserToGroupsMap =
   new HashMapString, ListString();
   private final long cacheTimeout;
   private final long negativeCacheTimeout;
   private final long warningDeltaMs;
   private final Timer timer;
+  private SetString negativeCache;
 
   public Groups(Configuration conf) {
 this(conf, new Timer());
@@ -99,11 +100,24 @@ public class Groups {
   .expireAfterWrite(10 * cacheTimeout, TimeUnit.MILLISECONDS)
   .build(new GroupCacheLoader());
 
+if(negativeCacheTimeout  0) {
+  CacheString, Boolean tempMap = CacheBuilder.newBuilder()
+.expireAfterWrite(negativeCacheTimeout, TimeUnit.MILLISECONDS)
+.ticker(new TimerToTickerAdapter(timer))
+.build();
+  negativeCache = Collections.newSetFromMap(tempMap.asMap());
+}
+
 if(LOG.isDebugEnabled())
   LOG.debug(Group mapping impl= + impl.getClass().getName() + 
   ; cacheTimeout= + cacheTimeout + ; warningDeltaMs= +
   warningDeltaMs);
   }
+  
+  @VisibleForTesting
+  SetString getNegativeCache() {
+return negativeCache;
+  }
 
   /*
* Parse the hadoop.user.group.static.mapping.overrides configuration to
@@ -159,13 +173,8 @@ public class Groups {
 
 // Check the negative cache first
 if (isNegativeCacheEnabled()) {
-  Long expirationTime = negativeCacheMask.get(user);
-  if (expirationTime != null) {
-if (timer.monotonicNow()  expirationTime) {
-  throw noGroupsForUser(user);
-} else {
-  negativeCacheMask.remove(user, expirationTime);
-}
+  if (negativeCache.contains(user)) {
+throw noGroupsForUser(user);
   }
 }
 
@@ -212,8 +221,7 @@ public class Groups {
 
   if (groups.isEmpty()) {
 if (isNegativeCacheEnabled()) {
-  long expirationTime = timer.monotonicNow() + negativeCacheTimeout;
-  negativeCacheMask.put(user, expirationTime);
+  negativeCache.add(user);
 }
 
 // We throw here to prevent Cache from retaining an empty group
@@ -252,7 +260,9 @@ public class Groups {
   LOG.warn(Error refreshing groups cache, e);
 }
 cache.invalidateAll();
-negativeCacheMask.clear();
+if(isNegativeCacheEnabled()) {
+  negativeCache.clear();
+}
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d740be6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java

hadoop git commit: HADOOP-11402. Negative user-to-group cache entries are never cleared for never-again-accessed users. Contributed by Varun Saxena.

2015-01-05 Thread benoy 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 94d342e60 - 53caeaa16


HADOOP-11402. Negative user-to-group cache entries are never cleared for 
never-again-accessed users. Contributed by Varun Saxena.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/53caeaa1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/53caeaa1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/53caeaa1

Branch: refs/heads/trunk
Commit: 53caeaa16b1450b54e994c77f5d0c8a767b88d57
Parents: 94d342e
Author: Benoy Antony be...@apache.org
Authored: Mon Jan 5 15:06:46 2015 -0800
Committer: Benoy Antony be...@apache.org
Committed: Mon Jan 5 15:06:46 2015 -0800

--
 .../java/org/apache/hadoop/security/Groups.java | 36 +--
 .../hadoop/security/TestGroupsCaching.java  | 48 
 2 files changed, 71 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/53caeaa1/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
index f3c5094..9fd39b0 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
@@ -23,12 +23,14 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
+import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Ticker;
 import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.Cache;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
 import org.apache.hadoop.HadoopIllegalArgumentException;
@@ -60,14 +62,13 @@ public class Groups {
   private final GroupMappingServiceProvider impl;
 
   private final LoadingCacheString, ListString cache;
-  private final ConcurrentHashMapString, Long negativeCacheMask =
-new ConcurrentHashMapString, Long();
   private final MapString, ListString staticUserToGroupsMap =
   new HashMapString, ListString();
   private final long cacheTimeout;
   private final long negativeCacheTimeout;
   private final long warningDeltaMs;
   private final Timer timer;
+  private SetString negativeCache;
 
   public Groups(Configuration conf) {
 this(conf, new Timer());
@@ -99,11 +100,24 @@ public class Groups {
   .expireAfterWrite(10 * cacheTimeout, TimeUnit.MILLISECONDS)
   .build(new GroupCacheLoader());
 
+if(negativeCacheTimeout  0) {
+  CacheString, Boolean tempMap = CacheBuilder.newBuilder()
+.expireAfterWrite(negativeCacheTimeout, TimeUnit.MILLISECONDS)
+.ticker(new TimerToTickerAdapter(timer))
+.build();
+  negativeCache = Collections.newSetFromMap(tempMap.asMap());
+}
+
 if(LOG.isDebugEnabled())
   LOG.debug(Group mapping impl= + impl.getClass().getName() + 
   ; cacheTimeout= + cacheTimeout + ; warningDeltaMs= +
   warningDeltaMs);
   }
+  
+  @VisibleForTesting
+  SetString getNegativeCache() {
+return negativeCache;
+  }
 
   /*
* Parse the hadoop.user.group.static.mapping.overrides configuration to
@@ -159,13 +173,8 @@ public class Groups {
 
 // Check the negative cache first
 if (isNegativeCacheEnabled()) {
-  Long expirationTime = negativeCacheMask.get(user);
-  if (expirationTime != null) {
-if (timer.monotonicNow()  expirationTime) {
-  throw noGroupsForUser(user);
-} else {
-  negativeCacheMask.remove(user, expirationTime);
-}
+  if (negativeCache.contains(user)) {
+throw noGroupsForUser(user);
   }
 }
 
@@ -212,8 +221,7 @@ public class Groups {
 
   if (groups.isEmpty()) {
 if (isNegativeCacheEnabled()) {
-  long expirationTime = timer.monotonicNow() + negativeCacheTimeout;
-  negativeCacheMask.put(user, expirationTime);
+  negativeCache.add(user);
 }
 
 // We throw here to prevent Cache from retaining an empty group
@@ -252,7 +260,9 @@ public class Groups {
   LOG.warn(Error refreshing groups cache, e);
 }
 cache.invalidateAll();
-negativeCacheMask.clear();
+if(isNegativeCacheEnabled()) {
+  negativeCache.clear();
+}
   }
 
   /**

hadoop git commit: HADOOP-11402. Negative user-to-group cache entries are never cleared for never-again-accessed users. Contributed by Varun Saxena.

2015-01-05 Thread benoy 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 d070597a0 - 0c2d996c2


HADOOP-11402. Negative user-to-group cache entries are never cleared for 
never-again-accessed users. Contributed by Varun Saxena.

(cherry picked from commit 53caeaa16b1450b54e994c77f5d0c8a767b88d57)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0c2d996c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0c2d996c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0c2d996c

Branch: refs/heads/branch-2
Commit: 0c2d996c2ce2f3e1c959a04f15bb6fb0ba9c4b1f
Parents: d070597
Author: Benoy Antony be...@apache.org
Authored: Mon Jan 5 15:06:46 2015 -0800
Committer: Benoy Antony be...@apache.org
Committed: Mon Jan 5 15:09:19 2015 -0800

--
 .../java/org/apache/hadoop/security/Groups.java | 36 +--
 .../hadoop/security/TestGroupsCaching.java  | 48 
 2 files changed, 71 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c2d996c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
index f3c5094..9fd39b0 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
@@ -23,12 +23,14 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
+import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Ticker;
 import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.Cache;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
 import org.apache.hadoop.HadoopIllegalArgumentException;
@@ -60,14 +62,13 @@ public class Groups {
   private final GroupMappingServiceProvider impl;
 
   private final LoadingCacheString, ListString cache;
-  private final ConcurrentHashMapString, Long negativeCacheMask =
-new ConcurrentHashMapString, Long();
   private final MapString, ListString staticUserToGroupsMap =
   new HashMapString, ListString();
   private final long cacheTimeout;
   private final long negativeCacheTimeout;
   private final long warningDeltaMs;
   private final Timer timer;
+  private SetString negativeCache;
 
   public Groups(Configuration conf) {
 this(conf, new Timer());
@@ -99,11 +100,24 @@ public class Groups {
   .expireAfterWrite(10 * cacheTimeout, TimeUnit.MILLISECONDS)
   .build(new GroupCacheLoader());
 
+if(negativeCacheTimeout  0) {
+  CacheString, Boolean tempMap = CacheBuilder.newBuilder()
+.expireAfterWrite(negativeCacheTimeout, TimeUnit.MILLISECONDS)
+.ticker(new TimerToTickerAdapter(timer))
+.build();
+  negativeCache = Collections.newSetFromMap(tempMap.asMap());
+}
+
 if(LOG.isDebugEnabled())
   LOG.debug(Group mapping impl= + impl.getClass().getName() + 
   ; cacheTimeout= + cacheTimeout + ; warningDeltaMs= +
   warningDeltaMs);
   }
+  
+  @VisibleForTesting
+  SetString getNegativeCache() {
+return negativeCache;
+  }
 
   /*
* Parse the hadoop.user.group.static.mapping.overrides configuration to
@@ -159,13 +173,8 @@ public class Groups {
 
 // Check the negative cache first
 if (isNegativeCacheEnabled()) {
-  Long expirationTime = negativeCacheMask.get(user);
-  if (expirationTime != null) {
-if (timer.monotonicNow()  expirationTime) {
-  throw noGroupsForUser(user);
-} else {
-  negativeCacheMask.remove(user, expirationTime);
-}
+  if (negativeCache.contains(user)) {
+throw noGroupsForUser(user);
   }
 }
 
@@ -212,8 +221,7 @@ public class Groups {
 
   if (groups.isEmpty()) {
 if (isNegativeCacheEnabled()) {
-  long expirationTime = timer.monotonicNow() + negativeCacheTimeout;
-  negativeCacheMask.put(user, expirationTime);
+  negativeCache.add(user);
 }
 
 // We throw here to prevent Cache from retaining an empty group
@@ -252,7 +260,9 @@ public class Groups {
   LOG.warn(Error refreshing groups cache, e);
 }
 cache.invalidateAll();
-negativeCacheMask.clear();
+if(isNegativeCacheEnabled()) {
+  negativeCache.clear();
+}
   }
 
   /**