Repository: incubator-ranger Updated Branches: refs/heads/master 02063004e -> ef1523191
Ranger-201:Addition changes to handle text file as source in addition to json Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/21152a57 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/21152a57 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/21152a57 Branch: refs/heads/master Commit: 21152a57cda53d064d86ae79b19ca2d53f8ab7fb Parents: 0206300 Author: rmani <[email protected]> Authored: Wed Jan 7 15:22:56 2015 -0800 Committer: rmani <[email protected]> Committed: Wed Jan 7 15:22:56 2015 -0800 ---------------------------------------------------------------------- src/main/assembly/usersync.xml | 1 + ugsync/pom.xml | 5 + .../config/UserGroupSyncConfig.java | 12 ++ .../process/FileSourceUserGroupBuilder.java | 124 ++++++++++--------- .../conf.dist/unixauthservice.properties | 6 + 5 files changed, 89 insertions(+), 59 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/21152a57/src/main/assembly/usersync.xml ---------------------------------------------------------------------- diff --git a/src/main/assembly/usersync.xml b/src/main/assembly/usersync.xml index 89f0dcc..0adb0a3 100644 --- a/src/main/assembly/usersync.xml +++ b/src/main/assembly/usersync.xml @@ -46,6 +46,7 @@ <include>org.apache.hadoop:hadoop-auth</include> <include>org.slf4j:slf4j-api</include> <include>org.apache.hadoop:hadoop-common</include> + <include>org.apache.commons:commons-csv</include> <include>org.apache.ranger:credentialbuilder</include> </includes> <unpack>false</unpack> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/21152a57/ugsync/pom.xml ---------------------------------------------------------------------- diff --git a/ugsync/pom.xml b/ugsync/pom.xml index c1a0193..93b812e 100644 --- a/ugsync/pom.xml +++ b/ugsync/pom.xml @@ -85,6 +85,11 @@ <version>${hadoop-auth.version}</version> </dependency> <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-csv</artifactId> + <version>1.0</version> + </dependency> + <dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-api</artifactId> <version>${slf4j-api.version}</version> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/21152a57/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java ---------------------------------------------------------------------- diff --git a/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java b/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java index 28372f1..181b107 100644 --- a/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java +++ b/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java @@ -48,6 +48,8 @@ public class UserGroupSyncConfig { public static final String UGSYNC_SOURCE_FILE_PROC = "usergroupSync.filesource.file"; + public static final String UGSYNC_SOURCE_FILE_DELIMITER = "usergroupSync.filesource.text.delimiter"; + private static final String SSL_KEYSTORE_PATH_PARAM = "keyStore" ; private static final String SSL_KEYSTORE_PATH_PASSWORD_PARAM = "keyStorePassword" ; @@ -111,6 +113,8 @@ public class UserGroupSyncConfig { private static final String DEFAULT_UGSYNC_GROUPNAME_CASE_CONVERSION_VALUE = UGSYNC_LOWER_CASE_CONVERSION_VALUE ; private static final String DEFAULT_USER_GROUP_NAME_ATTRIBUTE = "memberof,ismemberof"; + + private static final String DEFAULT_USER_GROUP_TEXTFILE_DELIMITER = ","; private Properties prop = new Properties() ; @@ -181,6 +185,14 @@ public class UserGroupSyncConfig { return val; } + public String getUserSyncFileSourceDelimiter(){ + String val = prop.getProperty(UGSYNC_SOURCE_FILE_DELIMITER) ; + if ( val == null) { + val = DEFAULT_USER_GROUP_TEXTFILE_DELIMITER; + } + return val; + } + public boolean isUserSyncEnabled() { String val = prop.getProperty(UGSYNC_ENABLED_PROP) ; return (val != null && val.trim().equalsIgnoreCase("true")) ; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/21152a57/ugsync/src/main/java/org/apache/ranger/unixusersync/process/FileSourceUserGroupBuilder.java ---------------------------------------------------------------------- diff --git a/ugsync/src/main/java/org/apache/ranger/unixusersync/process/FileSourceUserGroupBuilder.java b/ugsync/src/main/java/org/apache/ranger/unixusersync/process/FileSourceUserGroupBuilder.java index 65f8e6c..af18fa1 100644 --- a/ugsync/src/main/java/org/apache/ranger/unixusersync/process/FileSourceUserGroupBuilder.java +++ b/ugsync/src/main/java/org/apache/ranger/unixusersync/process/FileSourceUserGroupBuilder.java @@ -22,17 +22,21 @@ package org.apache.ranger.unixusersync.process; import java.io.BufferedReader; import java.io.File; import java.io.FileReader; -import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; import org.apache.log4j.Logger; import org.apache.ranger.unixusersync.config.UserGroupSyncConfig; import org.apache.ranger.usergroupsync.UserGroupSink; import org.apache.ranger.usergroupsync.UserGroupSource; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; import com.google.gson.stream.JsonReader; public class FileSourceUserGroupBuilder implements UserGroupSource { @@ -125,83 +129,85 @@ public class FileSourceUserGroupBuilder implements UserGroupSource { File f = new File(userGroupFilename); if (f.exists() && f.canRead()) { + Map<String,List<String>> tmpUser2GroupListMap = null; - Map<String,List<String>> tmpUser2GroupListMap = new HashMap<String,List<String>>(); - - JsonReader jsonReader = new JsonReader(new BufferedReader(new FileReader(f))); - - jsonReader.setLenient(true); - - jsonReader.beginArray(); - - while (jsonReader.hasNext() ) { - Map<String, List<String>> usergroupMap = getUserGroupMap(jsonReader); - - for(String user : usergroupMap.keySet()) { - List<String> groups = usergroupMap.get(user) ; - tmpUser2GroupListMap.put(user,groups); - } + if ( isJsonFile(userGroupFilename) ) { + tmpUser2GroupListMap = readJSONfile(f); + } else { + tmpUser2GroupListMap = readTextFile(f); } - - jsonReader.endArray(); - - jsonReader.close(); - user2GroupListMap = tmpUser2GroupListMap; - - usergroupFileModified = f.lastModified() ; - + if(tmpUser2GroupListMap != null) { + user2GroupListMap = tmpUser2GroupListMap; + + usergroupFileModified = f.lastModified() ; + } else { + LOG.info("No new UserGroup to sync at this time"); + } } else { throw new Exception("User Group Source File " + userGroupFilename + "doesn't not exist or readable"); } } + public boolean isJsonFile(String userGroupFilename) { + boolean ret = false; + + if ( userGroupFilename.toLowerCase().endsWith(".json")) { + ret = true; + } + + return ret; + } - public Map<String, List<String>> getUserGroupMap(JsonReader jsonReader) throws Exception { - + public Map<String, List<String>> readJSONfile(File jsonFile) throws Exception { Map<String, List<String>> ret = new HashMap<String, List<String>>(); - String user = null ; - List<String> groups = new ArrayList<String>(); - - jsonReader.beginObject(); - - while ( jsonReader.hasNext()) { - - String name = jsonReader.nextName(); - - if ( name.equals("user")) { - user = jsonReader.nextString(); - } else if ( name.equals("groups")) { - groups = getGroups(jsonReader); - } else { - StringBuilder sb = new StringBuilder(); - sb.append("User Group Source JSON array should have following **user** and **groups** as name tag e.g"); - sb.append("[ {\"user\":\"userid1\",\"groups\":[\"groupid1\",\"groupid2\"]},"); - sb.append("[ {\"user\":\"userid2\",\"groups\":[\"groupid1\",\"groupid2\"]}..]"); - throw new Exception(sb.toString()); - } - - if ( user != null ) { - ret.put(user, groups); - } - } + + JsonReader jsonReader = new JsonReader(new BufferedReader(new FileReader(jsonFile))); - jsonReader.endObject(); + Gson gson = new GsonBuilder().create() ; + + ret = gson.fromJson(jsonReader, ret.getClass()); return ret; + } - public List<String> getGroups(JsonReader reader) throws IOException { - List<String> ret = new ArrayList<String>(); + public Map<String, List<String>> readTextFile(File textFile) throws Exception { - reader.beginArray(); + Map<String, List<String>> ret = new HashMap<String, List<String>>(); - while(reader.hasNext()) { - ret.add(reader.nextString()); - } + String delimiter = config.getUserSyncFileSourceDelimiter(); + + CSVFormat csvFormat = CSVFormat.newFormat(delimiter.charAt(0)); - reader.endArray(); + CSVParser csvParser = new CSVParser(new BufferedReader(new FileReader(textFile)), csvFormat); + List<CSVRecord> csvRecordList = csvParser.getRecords(); + + if ( csvRecordList != null) { + for(CSVRecord csvRecord : csvRecordList) { + List<String> groups = new ArrayList<String>(); + String user = csvRecord.get(0); + if ( (user.substring(0,1).equals("\"")) && (user.substring(user.length()-1)).equals("\"")) { + user = user.substring(1,user.length()-1); + } + + int i = csvRecord.size(); + + for (int j = 1; j < i; j ++) { + String group = csvRecord.get(j); + if ( (group.substring(0,1).equals("\"")) && (group.substring(group.length()-1)).equals("\"")) { + group = group.substring(1,group.length()-1); + } + groups.add(group); + } + ret.put(user,groups); + } + } + + csvParser.close(); + return ret; } + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/21152a57/unixauthservice/conf.dist/unixauthservice.properties ---------------------------------------------------------------------- diff --git a/unixauthservice/conf.dist/unixauthservice.properties b/unixauthservice/conf.dist/unixauthservice.properties index 83282ed..680b53f 100644 --- a/unixauthservice/conf.dist/unixauthservice.properties +++ b/unixauthservice/conf.dist/unixauthservice.properties @@ -85,8 +85,14 @@ usergroupSync.source.impl.class = # The following properties are relevant # only if value of usergroupSync.source.impl.class is # org.apache.ranger.unixusersync.process.FileSourceUserGroupBuilder +# e.g usergroupSync.filesource.file = /tmp/usergroup.json or /tmp/usergroup.txt +# if the file is .txt usergroupSync.filesource.text.delimiter will be used. +# if the file is .json JSONParser will be used instead of delimiter. +# default delimiter value : , +# e.g To input a tab delimited file use usergroupSync.filesource.text.delimiter = \t # --------------------------------------------------------------- usergroupSync.filesource.file = +usergroupSync.filesource.text.delimiter = , # --------------------------------------------------------------- # The following properties are relevant
