Repository: incubator-ranger
Updated Branches:
  refs/heads/master 0966bbccb -> 658f2310d


Issue RANGER-827

Signed-off-by: Velmurugan Periasamy <[email protected]>


Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/658f2310
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/658f2310
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/658f2310

Branch: refs/heads/master
Commit: 658f2310dc674391b7d930e4d3a43cc976693370
Parents: 0966bbc
Author: Bolke de Bruin <[email protected]>
Authored: Sun Feb 7 22:48:07 2016 +0100
Committer: Velmurugan Periasamy <[email protected]>
Committed: Thu Feb 11 17:49:06 2016 -0500

----------------------------------------------------------------------
 .../config/UserGroupSyncConfig.java             |  67 ++-
 .../process/UnixUserGroupBuilder.java           | 454 +++++++++++++------
 .../process/UnixUserGroupBuilderTest.java       | 108 +++++
 3 files changed, 481 insertions(+), 148 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/658f2310/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java
----------------------------------------------------------------------
diff --git 
a/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java
 
b/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java
index e46b469..9bcf767 100644
--- 
a/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java
+++ 
b/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java
@@ -58,6 +58,8 @@ public class UserGroupSyncConfig  {
        public static final String  UGSYNC_PM_URL_PROP =        
"ranger.usersync.policymanager.baseURL" ;
        
        public static final String  UGSYNC_MIN_USERID_PROP  =   
"ranger.usersync.unix.minUserId" ;
+
+       public static final String  UGSYNC_MIN_GROUPID_PROP =   
"ranger.usersync.unix.minGroupId" ;
        
        public static final String  UGSYNC_MAX_RECORDS_PER_API_CALL_PROP  =     
"ranger.usersync.policymanager.maxrecordsperapicall" ;
 
@@ -162,6 +164,16 @@ public class UserGroupSyncConfig  {
   private static final String LGSYNC_GROUP_MEMBER_ATTRIBUTE_NAME = 
"ranger.usersync.group.memberattributename";
   private static final String DEFAULT_LGSYNC_GROUP_MEMBER_ATTRIBUTE_NAME = 
"member";
 
+       private static final String UGSYNC_UPDATE_MILLIS_MIN = 
"ranger.usersync.unix.updatemillismin";
+       private final static long DEFAULT_UGSYNC_UPDATE_MILLIS_MIN = 1 * 60 * 
1000; // ms
+
+       private static final String UGSYNC_UNIX_BACKEND = 
"ranger.usersync.unix.backend";
+       private final static String DEFAULT_UGSYNC_UNIX_BACKEND = "passwd";
+
+       private static final String UGSYNC_GROUP_ENUMERATE_ENABLED = 
"ranger.usersync.group.enumerate";
+
+       private static final String UGSYNC_GROUP_ENUMERATE_GROUPS = 
"ranger.usersync.group.enumerategroup";
+
        private static final String SYNC_POLICY_MGR_KEYSTORE = 
"ranger.usersync.policymgr.keystore";
 
        private static final String SYNC_POLICY_MGR_ALIAS = 
"ranger.usersync.policymgr.alias";
@@ -192,7 +204,7 @@ public class UserGroupSyncConfig  {
        private static volatile UserGroupSyncConfig me = null ;
        
        public static UserGroupSyncConfig getInstance() {
-        UserGroupSyncConfig result = me;
+               UserGroupSyncConfig result = me;
                if (result == null) {
                        synchronized(UserGroupSyncConfig.class) {
                                result = me ;
@@ -204,11 +216,10 @@ public class UserGroupSyncConfig  {
                return result ;
        }
        
-       
        private UserGroupSyncConfig() {
-               init() ;
+               init();
        }
-       
+
        private void init() {
                readConfigFile(CONFIG_FILE);
                readConfigFile(DEFAULT_CONFIG_FILE);
@@ -321,19 +332,35 @@ public class UserGroupSyncConfig  {
                }
                return val;
        }
-       
+
+       public String getUnixBackend() {
+               String val = prop.getProperty(UGSYNC_UNIX_BACKEND);
+               if ( val == null ) {
+                       val = DEFAULT_UGSYNC_UNIX_BACKEND;
+               }
+
+               return val;
+       }
+
        public boolean isUserSyncEnabled() {
                String val = prop.getProperty(UGSYNC_ENABLED_PROP) ;
                return (val != null && val.trim().equalsIgnoreCase("true")) ;
        }
 
-       
+       public String getEnumerateGroups() {
+               return prop.getProperty(UGSYNC_GROUP_ENUMERATE_GROUPS);
+       }
+
+       public boolean isGroupEnumerateEnabled() {
+               String val = prop.getProperty(UGSYNC_GROUP_ENUMERATE_ENABLED) ;
+               return (val != null && val.trim().equalsIgnoreCase("true")) ;
+       }
+
        public boolean isMockRunEnabled() {
                String val = prop.getProperty(UGSYNC_MOCK_RUN_PROP) ;
                return (val != null && val.trim().equalsIgnoreCase("true")) ;
        }
-       
-       
+
        public String getPolicyManagerBaseURL() {
                return prop.getProperty(UGSYNC_PM_URL_PROP) ;
        }
@@ -342,6 +369,8 @@ public class UserGroupSyncConfig  {
        public String getMinUserId() {
                return prop.getProperty(UGSYNC_MIN_USERID_PROP) ;
        }
+
+       public String getMinGroupId() { return 
prop.getProperty(UGSYNC_MIN_GROUPID_PROP) ; }
        
        public String getMaxRecordsPerAPICall() {
                return prop.getProperty(UGSYNC_MAX_RECORDS_PER_API_CALL_PROP) ;
@@ -366,7 +395,20 @@ public class UserGroupSyncConfig  {
                return  prop.getProperty(SSL_TRUSTSTORE_PATH_PASSWORD_PARAM) ;
        }
        
-       
+       public long getUpdateMillisMin() {
+               String val = prop.getProperty(UGSYNC_UPDATE_MILLIS_MIN) ;
+               if (val == null) {
+                       return DEFAULT_UGSYNC_UPDATE_MILLIS_MIN ;
+               }
+
+               long ret = Long.parseLong(val) ;
+               if (ret < DEFAULT_UGSYNC_UPDATE_MILLIS_MIN) {
+                       return DEFAULT_UGSYNC_UPDATE_MILLIS_MIN;
+               }
+
+               return ret;
+       }
+
        public long getSleepTimeInMillisBetweenCycle() throws Throwable {
                String val =  
prop.getProperty(UGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_PARAM) ;
                if (val == null) {
@@ -789,7 +831,7 @@ public class UserGroupSyncConfig  {
                }
                return val;
        }
-       
+
        /* Used only for unit testing */
     public void setUserSearchFilter(String filter) {
             prop.setProperty(LGSYNC_USER_SEARCH_FILTER, filter);
@@ -809,4 +851,9 @@ public class UserGroupSyncConfig  {
     public void setPagedResultsEnabled(boolean pagedResultsEnabled) {
         prop.setProperty(LGSYNC_PAGED_RESULTS_ENABLED, 
String.valueOf(pagedResultsEnabled));
     }
+
+       /* Used only for unit testing */
+       public void setProperty(String name, String value) {
+               prop.setProperty(name, value);
+       }
 }

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/658f2310/ugsync/src/main/java/org/apache/ranger/unixusersync/process/UnixUserGroupBuilder.java
----------------------------------------------------------------------
diff --git 
a/ugsync/src/main/java/org/apache/ranger/unixusersync/process/UnixUserGroupBuilder.java
 
b/ugsync/src/main/java/org/apache/ranger/unixusersync/process/UnixUserGroupBuilder.java
index f0a29f4..780f670 100644
--- 
a/ugsync/src/main/java/org/apache/ranger/unixusersync/process/UnixUserGroupBuilder.java
+++ 
b/ugsync/src/main/java/org/apache/ranger/unixusersync/process/UnixUserGroupBuilder.java
@@ -17,16 +17,15 @@
  * under the License.
  */
 
- package org.apache.ranger.unixusersync.process;
+package org.apache.ranger.unixusersync.process;
 
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileReader;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.io.InputStreamReader;
+import java.util.*;
 
+import com.google.common.annotations.VisibleForTesting;
 import org.apache.log4j.Logger;
 import org.apache.ranger.unixusersync.config.UserGroupSyncConfig;
 import org.apache.ranger.usergroupsync.UserGroupSink;
@@ -35,51 +34,94 @@ import org.apache.ranger.usergroupsync.UserGroupSource;
 public class UnixUserGroupBuilder implements UserGroupSource {
        
        private static final Logger LOG = 
Logger.getLogger(UnixUserGroupBuilder.class) ;
+       private final static String OS = System.getProperty("os.name") ;
 
+       // kept for legacy support
        public static final String UNIX_USER_PASSWORD_FILE = "/etc/passwd" ;
        public static final String UNIX_GROUP_FILE = "/etc/group" ;
 
+       /** Shell commands to get users and groups */
+       static final String LINUX_GET_ALL_USERS_CMD = "getent passwd" ;
+       static final String LINUX_GET_ALL_GROUPS_CMD = "getent group" ;
+       static final String LINUX_GET_GROUP_CMD = "getent group %s" ;
+
+       // mainly for testing purposes
+       // there might be a better way
+       static final String MAC_GET_ALL_USERS_CMD = "dscl . -readall /Users 
UniqueID PrimaryGroupID | " +
+                       "awk 'BEGIN { OFS = \":\"; ORS=\"\\n\"; i=0;}" +
+                       "/RecordName: / {name = $2;i = 0;}/PrimaryGroupID: / 
{gid = $2;}" +
+                       "/^ / {if (i == 0) { i++; name = $1;}}" +
+                       "/UniqueID: / {uid = $2;print name, \"*\", gid, uid;}'" 
;
+       static final String MAC_GET_ALL_GROUPS_CMD = "dscl . -list /Groups 
PrimaryGroupID | " +
+                       "awk -v OFS=\":\" '{print $1, \"*\", $2, \"\"}'" ;
+       static final String MAC_GET_GROUP_CMD = "dscl . -read /Groups/%1$s | 
paste -d, -s - | sed -e 's/:/|/g' | " +
+                       "awk -v OFS=\":\" -v ORS=\"\\n\" -F, '{print 
\"%1$s\",\"*\",$6,$4}' | " +
+                       "sed -e 's/:[^:]*| /:/g' | sed -e 's/ /,/g'" ;
+
+       static final String BACKEND_PASSWD = "passwd";
+
+       private boolean enumerateGroupMembers = false;
+       private boolean useNss = false;
+
+       private long lastUpdateTime = 0; // Last time maps were updated
+       private long timeout = 0;
+
        private UserGroupSyncConfig config = UserGroupSyncConfig.getInstance() ;
        private Map<String,List<String>>        user2GroupListMap = new 
HashMap<String,List<String>>();
        private Map<String,List<String>>        internalUser2GroupListMap = new 
HashMap<String,List<String>>();
        private Map<String,String>                      groupId2groupNameMap = 
new HashMap<String,String>() ;
        private int                                             minimumUserId  
= 0 ;
-       
-       private long  passwordFileModiiedAt = 0 ;
-       private long  groupFileModifiedAt = 0 ;
-               
-       
+       private int                                                     
minimumGroupId = 0 ;
+
+       private long passwordFileModifiedAt = 0 ;
+       private long groupFileModifiedAt = 0 ;
+
        public static void main(String[] args) throws Throwable {
-               UnixUserGroupBuilder  ugbuilder = new UnixUserGroupBuilder() ;
+               UnixUserGroupBuilder ugbuilder = new UnixUserGroupBuilder() ;
                ugbuilder.init();
                ugbuilder.print(); 
        }
        
        public UnixUserGroupBuilder() {
                minimumUserId = Integer.parseInt(config.getMinUserId()) ;
-               LOG.debug("Minimum UserId: " + minimumUserId) ;
+               minimumGroupId = Integer.parseInt(config.getMinGroupId()) ;
+
+               LOG.debug("Minimum UserId: " + minimumUserId + ", minimum 
GroupId: " + minimumGroupId) ;
+
+               timeout = config.getUpdateMillisMin() ;
+               enumerateGroupMembers = config.isGroupEnumerateEnabled();
+
+               if (!config.getUnixBackend().equalsIgnoreCase(BACKEND_PASSWD)) {
+                       useNss = true;
+               } else {
+                       LOG.warn("DEPRECATED: Unix backend is configured to use 
/etc/passwd and /etc/group files directly " +
+                                       "instead of standard system 
mechanisms.");
+               }
        }
 
        @Override
        public void init() throws Throwable {
                buildUserGroupInfo() ;
        }
-       
+
        @Override
        public boolean isChanged() {
-               long TempPasswordFileModiiedAt = new 
File(UNIX_USER_PASSWORD_FILE).lastModified() ;
-               if (passwordFileModiiedAt != TempPasswordFileModiiedAt) {
+               if (useNss)
+                       return System.currentTimeMillis() - lastUpdateTime > 
timeout ;
+
+               long TempPasswordFileModifiedAt = new 
File(UNIX_USER_PASSWORD_FILE).lastModified() ;
+               if (passwordFileModifiedAt != TempPasswordFileModifiedAt) {
                        return true ;
                }
-               
+
                long TempGroupFileModifiedAt = new 
File(UNIX_GROUP_FILE).lastModified() ;
                if (groupFileModifiedAt != TempGroupFileModifiedAt) {
                        return true ;
                }
-               
+
                return false ;
        }
-       
+
 
        @Override
        public void updateSink(UserGroupSink sink) throws Throwable {
@@ -96,10 +138,21 @@ public class UnixUserGroupBuilder implements 
UserGroupSource {
        
        private void buildUserGroupInfo() throws Throwable {
                user2GroupListMap = new HashMap<String,List<String>>();
-               groupId2groupNameMap = new HashMap<String,String>() ;
+               groupId2groupNameMap = new HashMap<String, String>();
+
+               if (OS.startsWith("Mac")) {
+                       buildUnixGroupList(MAC_GET_ALL_GROUPS_CMD, 
MAC_GET_GROUP_CMD, false);
+                       buildUnixUserList(MAC_GET_ALL_USERS_CMD);
+               } else {
+                       if (!OS.startsWith("Linux")) {
+                               LOG.warn("Platform not recognized assuming 
Linux compatible");
+                       }
+                       buildUnixGroupList(LINUX_GET_ALL_GROUPS_CMD, 
LINUX_GET_GROUP_CMD, true);
+                       buildUnixUserList(LINUX_GET_ALL_USERS_CMD);
+               }
+
+               lastUpdateTime = System.currentTimeMillis() ;
 
-               buildUnixGroupList(); 
-               buildUnixUserList();
                if (LOG.isDebugEnabled()) {
                        print() ;
                }
@@ -117,139 +170,264 @@ public class UnixUserGroupBuilder implements 
UserGroupSource {
                }
        }
        
-       private void buildUnixUserList() throws Throwable {
-               
-               File f = new File(UNIX_USER_PASSWORD_FILE) ;
-
-               if (f.exists()) {
-                       
-                       
-                       BufferedReader reader = null ;
-                       
-                       reader = new BufferedReader(new FileReader(f)) ;
-                       
-                       String line = null ;
-                       
-                       while ( (line = reader.readLine()) != null) {
-                               
-                               if (line.trim().isEmpty()) 
-                                       continue ;
-                               
-                               String[] tokens = line.split(":") ;
-                               
-                               int len = tokens.length ;
-                               
-                               if (len < 2) {
-                                       continue ;
+       private void buildUnixUserList(String command) throws Throwable {
+               BufferedReader reader = null;
+
+               if (!useNss) {
+                       File file = new File(UNIX_USER_PASSWORD_FILE);
+                       passwordFileModifiedAt = file.lastModified();
+                       reader = new BufferedReader(new FileReader(file)) ;
+               } else {
+                       Process process = Runtime.getRuntime().exec(
+                                       new String[]{"bash", "-c", command});
+
+                       reader = new BufferedReader(new 
InputStreamReader(process.getInputStream()));
+               }
+
+               String line = null;
+               Map<String,String> userName2uid = new HashMap<String,String>();
+
+               while ((line = reader.readLine()) != null) {
+                       if (line.trim().isEmpty())
+                               continue;
+
+                       String[] tokens = line.split(":");
+
+                       int len = tokens.length;
+
+                       if (len < 2) {
+                               continue;
+                       }
+
+                       String userName = tokens[0];
+                       String userId = tokens[2];
+                       String groupId = tokens[3];
+
+                       int numUserId = -1;
+
+                       try {
+                               numUserId = Integer.parseInt(userId);
+                       } catch (NumberFormatException nfe) {
+                               LOG.warn("Unix UserId: [" + userId + "]: can 
not be parsed as valid int. considering as  -1.", nfe);
+                               numUserId = -1;
+                       }
+
+                       if (numUserId >= minimumUserId) {
+                               userName2uid.put(userName, userId);
+                               String groupName = 
groupId2groupNameMap.get(groupId);
+                               if (groupName != null) {
+                                       List<String> groupList = new 
ArrayList<String>();
+                                       groupList.add(groupName);
+                                       // do we already know about this use's 
membership to other groups?  If so add those, too
+                                       if 
(internalUser2GroupListMap.containsKey(userName)) {
+                                               List<String> map = 
internalUser2GroupListMap.get(userName);
+
+                                               // there could be duplicates
+                                               map.remove(groupName);
+                                               groupList.addAll(map);
+                                       }
+                                       user2GroupListMap.put(userName, 
groupList);
+                               } else {
+                                       // we are ignoring the possibility that 
this user was present in /etc/groups.
+                                       LOG.warn("Group Name could not be found 
for group id: [" + groupId + "]. Skipping adding user [" + userName + "] with 
id [" + userId + "].");
                                }
-                               
-                               String userName = tokens[0] ;
-                               String userId = tokens[2] ;
-                               String groupId = tokens[3] ;
-                               
-                               int numUserId = -1 ; 
-                               
+                       } else {
+                               LOG.debug("Skipping user [" + userName + "] 
since its userid [" + userId + "] is less than minuserid limit [" + 
minimumUserId + "].");
+                       }
+               }
+
+               reader.close();
+
+               if (!useNss)
+                       return;
+
+               // this does a reverse check as not all users might be listed 
in getent passwd
+               if (enumerateGroupMembers) {
+                       LOG.debug("Start drill down group members");
+                       for (Map.Entry<String, List<String>> entry : 
internalUser2GroupListMap.entrySet()) {
+                               // skip users we already now about
+                               if 
(user2GroupListMap.containsKey(entry.getKey()))
+                                       continue;
+
+                               LOG.debug("Enumerating user " + entry.getKey());
+
+                               int numUserId = -1;
                                try {
-                                       numUserId = Integer.parseInt(userId) ;
-                               }
-                               catch(NumberFormatException nfe) {
-                                       LOG.warn("Unix UserId: [" + userId + 
"]: can not be parsed as valid int. considering as  -1.", nfe);
-                                       numUserId = -1 ;
+                                       numUserId = 
Integer.parseInt(userName2uid.get(entry.getKey()));
+                               } catch (NumberFormatException nfe) {
+                                       numUserId = -1;
                                }
-                                                                       
-                               if (numUserId >= minimumUserId ) {
-                                       String groupName = 
groupId2groupNameMap.get(groupId) ;
-                                       if (groupName != null) {
-                                               List<String> groupList = new 
ArrayList<String>();
-                                               groupList.add(groupName);
-                                               // do we already know about 
this use's membership to other groups?  If so add those, too
-                                               if 
(internalUser2GroupListMap.containsKey(userName)) {
-                                                       
groupList.addAll(internalUser2GroupListMap.get(userName));
-                                               }
-                                               user2GroupListMap.put(userName, 
groupList);
-                                       }
-                                       else {
-                                               // we are ignoring the 
possibility that this user was present in /etc/groups.
-                                               LOG.warn("Group Name could not 
be found for group id: [" + groupId + "]. Skipping adding user [" + userName + 
"] with id [" + userId + "].") ;
-                                       }
+
+                               // if a user comes from an external group we 
might not have a uid
+                               if (numUserId < minimumUserId && numUserId != 
-1)
+                                       continue;
+
+
+                               // "id" is same across Linux / BSD / MacOSX
+                               // gids are used as id might return groups with 
spaces, ie "domain users"
+                               Process process = Runtime.getRuntime().exec(
+                                               new String[]{"bash", "-c", "id 
-G " + entry.getKey()});
+
+                               reader = new BufferedReader(new 
InputStreamReader(process.getInputStream()));
+                               line = reader.readLine();
+                               reader.close();
+
+                               LOG.debug("id -G returned " + line);
+
+                               if (line.trim().isEmpty()) {
+                                       LOG.warn("User " + entry.getKey() + " 
could not be resolved");
+                                       continue;
                                }
-                               else {
-                                       LOG.debug("Skipping user [" + userName 
+ "] since its userid [" + userId + "] is less than minuserid limit [" + 
minimumUserId + "].");
+
+                               String[] gids = line.split(" ");
+
+                               // check if all groups returned by id are 
visible to ranger
+                               ArrayList<String> allowedGroups = new 
ArrayList<String>();
+                               for (String gid : gids) {
+                                       int numGroupId = Integer.parseInt(gid);
+                                       if (numGroupId < minimumGroupId)
+                                               continue;
+
+                                       String groupName = 
groupId2groupNameMap.get(gid);
+                                       if (groupName != null)
+                                               allowedGroups.add(groupName);
                                }
-                       }
-                       
-                       reader.close() ;
-                       
-                       passwordFileModiiedAt = f.lastModified() ;
 
+                               user2GroupListMap.put(entry.getKey(), 
allowedGroups);
+                       }
+                       LOG.debug("End drill down group members");
                }
        }
 
-       
-       private void buildUnixGroupList() throws Throwable {
-               
-               File f = new File(UNIX_GROUP_FILE) ;
-               
-               if (f.exists()) {
-                       
-                       BufferedReader reader = null ;
-                       
-                       reader = new BufferedReader(new FileReader(f)) ;
-                       
-                       String line = null ;
-                       
-                       
-                       
-                       while ( (line = reader.readLine()) != null) {
-
-                               if (line.trim().isEmpty()) 
-                                       continue ;
-                               
-                               String[] tokens = line.split(":") ;
-                               
-                               int len = tokens.length ;
-                               
-                               if (len < 2) {
-                                       continue ;
-                               }
-                               
-                               String groupName = tokens[0] ;
-                               String groupId = tokens[2] ;
-                               String groupMembers = null ;
-                               
-                               if (tokens.length > 3) {
-                                       groupMembers = tokens[3] ;
+       private void parseMembers(String line) {
+               if (line == null || line.isEmpty())
+                       return;
+
+               String[] tokens = line.split(":");
+
+               if (tokens.length < 2)
+                       return;
+
+               String groupName = tokens[0];
+               String groupId = tokens[2];
+               String groupMembers = null;
+
+               if (tokens.length > 3)
+                       groupMembers = tokens[3];
+
+               if (groupId2groupNameMap.containsKey(groupId)) {
+                       groupId2groupNameMap.remove(groupId);
+               }
+
+               int numGroupId = Integer.parseInt(groupId);
+               if (numGroupId < minimumGroupId)
+                       return;
+
+               groupId2groupNameMap.put(groupId, groupName);
+
+               if (groupMembers != null && !groupMembers.trim().isEmpty()) {
+                       for (String user : groupMembers.split(",")) {
+                               List<String> groupList = 
internalUser2GroupListMap.get(user);
+                               if (groupList == null) {
+                                       groupList = new ArrayList<String>();
+                                       internalUser2GroupListMap.put(user, 
groupList);
                                }
-                               
-                               if (groupId2groupNameMap.containsKey(groupId)) {
-                                       groupId2groupNameMap.remove(groupId) ;
+                               if (!groupList.contains(groupName)) {
+                                       groupList.add(groupName);
                                }
-                               
-                               groupId2groupNameMap.put(groupId,groupName) ;
-                               // also build an internal map of users to their 
group list which is consulted by user list creator
-                               if (groupMembers != null && ! 
groupMembers.trim().isEmpty()) {
-                                       for(String user : 
groupMembers.split(",")) {
-                                               List<String> groupList = 
internalUser2GroupListMap.get(user) ;
-                                               if (groupList == null) {
-                                                       groupList = new 
ArrayList<String>() ;
-                                                       
internalUser2GroupListMap.put(user, groupList) ;
-                                               }
-                                               if (! 
groupList.contains(groupName)) {
-                                                       
groupList.add(groupName) ;
-                                               }
-                                       }
+                       }
+               }
+       }
+
+       private void buildUnixGroupList(String allGroupsCmd, String groupCmd, 
boolean useGid) throws Throwable {
+               LOG.debug("Start enumerating groups");
+               BufferedReader reader;
+
+               if (!useNss) {
+                       File file = new File(UNIX_GROUP_FILE);
+                       groupFileModifiedAt = file.lastModified();
+                       reader = new BufferedReader(new FileReader(file)) ;
+               } else {
+                       Process process = Runtime.getRuntime().exec(
+                                       new String[]{"bash", "-c", 
allGroupsCmd});
+                       reader = new BufferedReader(new 
InputStreamReader(process.getInputStream()));
+               }
+
+               String line = null;
+
+               while ((line = reader.readLine()) != null) {
+                       if (line.trim().isEmpty())
+                               continue;
+
+                       parseMembers(line);
+               }
+
+               reader.close();
+
+               LOG.debug("End enumerating group");
+
+               if (!useNss)
+                       return;
+
+               if (enumerateGroupMembers) {
+                       LOG.debug("Start enumerating group members");
+                       Map<String,String> copy = new HashMap<String, 
String>(groupId2groupNameMap);
+
+                       for (Map.Entry<String, String> group : copy.entrySet()) 
{
+                               LOG.debug("Enumerating group: " + 
group.getValue() + " GID(" + group.getKey() + ")");
+
+                               String command;
+                               if (useGid) {
+                                       command = String.format(groupCmd, 
group.getKey());
+                               } else {
+                                       command = String.format(groupCmd, 
group.getValue());
                                }
 
-                               
+                               String[] cmd = new String[]{"bash", "-c", 
command + " " + group.getKey()};
+                               LOG.debug("Executing: " + Arrays.toString(cmd));
+
+                               Process process = 
Runtime.getRuntime().exec(cmd);
+                               reader = new BufferedReader(new 
InputStreamReader(process.getInputStream()));
+                               line = reader.readLine();
+                               reader.close();
+
+                               LOG.debug("bash -c " + command + " for group " 
+ group + " returned " + line);
+
+                               parseMembers(line);
                        }
-                       
-                       reader.close() ;
-                       
-                       groupFileModifiedAt = f.lastModified() ;
+                       LOG.debug("End enumerating group members");
+               }
+
+               if (config.getEnumerateGroups() != null) {
+                       String[] groups = 
config.getEnumerateGroups().split(",");
+
+                       LOG.debug("Adding extra groups");
+
+                       for (String group : groups) {
+                               String command = groupCmd.format(group);
+                               String[] cmd = new String[]{"bash", "-c", 
command + " '" + group + "'"};
+                               LOG.debug("Executing: " + Arrays.toString(cmd));
+
+                               Process process = 
Runtime.getRuntime().exec(cmd);
+                               reader = new BufferedReader(new 
InputStreamReader(process.getInputStream()));
+                               line = reader.readLine();
+                               reader.close();
+                               LOG.debug("bash -c " + command + " for group " 
+ group + " returned " + line);
 
-               
+                               parseMembers(line);
+                       }
+                       LOG.debug("Done adding extra groups");
                }
        }
-       
+
+       @VisibleForTesting
+       Map<String,List<String>> getUser2GroupListMap() {
+               return user2GroupListMap;
+       }
+
+       @VisibleForTesting
+       Map<String,String> getGroupId2groupNameMap() {
+               return groupId2groupNameMap;
+       }
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/658f2310/ugsync/src/test/java/org/apache/ranger/unixusersync/process/UnixUserGroupBuilderTest.java
----------------------------------------------------------------------
diff --git 
a/ugsync/src/test/java/org/apache/ranger/unixusersync/process/UnixUserGroupBuilderTest.java
 
b/ugsync/src/test/java/org/apache/ranger/unixusersync/process/UnixUserGroupBuilderTest.java
new file mode 100644
index 0000000..e4d5456
--- /dev/null
+++ 
b/ugsync/src/test/java/org/apache/ranger/unixusersync/process/UnixUserGroupBuilderTest.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.unixusersync.process;
+
+import org.apache.ranger.unixusersync.config.UserGroupSyncConfig;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.*;
+import static org.hamcrest.Matchers.*;
+
+public class UnixUserGroupBuilderTest {
+    private UserGroupSyncConfig config;
+
+    @Before
+    public void setUp() throws Exception {
+        config = UserGroupSyncConfig.getInstance();
+        config.setProperty("ranger.usersync.unix.minUserId", "0");
+        config.setProperty("ranger.usersync.unix.minGroupId", "0");
+    }
+
+    @Test
+    public void testBuilderPasswd() throws Throwable {
+        config.setProperty("ranger.usersync.unix.backend", "passwd");
+
+        UnixUserGroupBuilder builder = new UnixUserGroupBuilder();
+        builder.init();
+
+        Map<String, String> groups = builder.getGroupId2groupNameMap();
+        String name = groups.get("0");
+        assertThat(name, anyOf(equalTo("wheel"), equalTo("root")));
+
+        Map<String, List<String>> users = builder.getUser2GroupListMap();
+        List<String> usergroups = users.get("root");
+        assertNotNull(usergroups);
+        assertThat(usergroups, anyOf(hasItem("wheel"), hasItem("root")));
+
+    }
+
+    @Test
+    public void testBuilderNss() throws Throwable {
+        config.setProperty("ranger.usersync.unix.backend", "nss");
+
+        UnixUserGroupBuilder builder = new UnixUserGroupBuilder();
+        builder.init();
+
+        Map<String, String> groups = builder.getGroupId2groupNameMap();
+        String name = groups.get("0");
+        assertThat(name, anyOf(equalTo("wheel"), equalTo("root")));
+
+        Map<String, List<String>> users = builder.getUser2GroupListMap();
+        List<String> usergroups = users.get("root");
+        assertNotNull(usergroups);
+        assertThat(usergroups, anyOf(hasItem("wheel"), hasItem("root")));
+    }
+
+    @Test
+    public void testBuilderExtraGroups() throws Throwable {
+        config.setProperty("ranger.usersync.unix.backend", "nss");
+        config.setProperty("ranger.usersync.group.enumerategroup", 
"root,wheel,daemon");
+
+        UnixUserGroupBuilder builder = new UnixUserGroupBuilder();
+        builder.init();
+
+        // this is not a full test as it cannot be mocked sufficiently
+        Map<String, String> groups = builder.getGroupId2groupNameMap();
+        assertTrue(groups.containsValue("daemon"));
+        assertThat(groups, anyOf(hasValue("wheel"), hasValue("root")));
+    }
+
+    @Test
+    public void testMinUidGid() throws Throwable {
+        config.setProperty("ranger.usersync.unix.backend", "nss");
+        config.setProperty("ranger.usersync.unix.minUserId", "500");
+        config.setProperty("ranger.usersync.unix.minGroupId", "500");
+
+        UnixUserGroupBuilder builder = new UnixUserGroupBuilder();
+        builder.init();
+
+        // this is not a full test as it cannot be mocked sufficiently
+        Map<String, String> groups = builder.getGroupId2groupNameMap();
+        assertFalse(groups.containsValue("wheel"));
+
+        Map<String, List<String>> users = builder.getUser2GroupListMap();
+        assertNull(users.get("root"));
+    }
+
+}

Reply via email to