mneethiraj commented on code in PR #440:
URL: https://github.com/apache/ranger/pull/440#discussion_r1879004080
##########
ugsync-util/pom.xml:
##########
@@ -26,6 +26,9 @@
<packaging>jar</packaging>
<name>User Group Synchronizer Util</name>
<description>User Group Synchronizer Util</description>
+ <properties>
+ <checkstyle.failOnViolation>true</checkstyle.failOnViolation>
Review Comment:
`<checkstyle.skip>false</checkstyle.skip>` would be needed as well.
##########
ugsync/src/main/java/org/apache/ranger/ldapusersync/process/CustomSSLSocketFactory.java:
##########
@@ -30,89 +42,70 @@
import java.security.KeyStore;
import java.security.SecureRandom;
-import javax.net.ssl.KeyManager;
-import javax.net.ssl.KeyManagerFactory;
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.SSLSocketFactory;
-import javax.net.ssl.TrustManager;
-import javax.net.ssl.TrustManagerFactory;
-
-import org.apache.ranger.unixusersync.config.UserGroupSyncConfig;
-import org.apache.ranger.unixusersync.process.PolicyMgrUserGroupBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class CustomSSLSocketFactory extends SSLSocketFactory{
- private static final Logger LOG =
LoggerFactory.getLogger(CustomSSLSocketFactory.class);
- private SSLSocketFactory sockFactory;
- private UserGroupSyncConfig config = UserGroupSyncConfig.getInstance();
+public class CustomSSLSocketFactory extends SSLSocketFactory {
+ private static final Logger LOG =
LoggerFactory.getLogger(CustomSSLSocketFactory.class);
+ private SSLSocketFactory sockFactory;
+ private UserGroupSyncConfig config =
UserGroupSyncConfig.getInstance();
public CustomSSLSocketFactory() {
- SSLContext sslContext = null;
- String keyStoreFile = config.getSSLKeyStorePath();
- String keyStoreFilepwd = config.getSSLKeyStorePathPassword();
- String trustStoreFile = config.getSSLTrustStorePath();
- String trustStoreFilepwd = config.getSSLTrustStorePathPassword();
- String keyStoreType = config.getSSLKeyStoreType();
- String trustStoreType = config.getSSLTrustStoreType();
- try {
-
- KeyManager[] kmList = null;
- TrustManager[] tmList = null;
-
- if (keyStoreFile != null && keyStoreFilepwd != null) {
-
- KeyStore keyStore =
KeyStore.getInstance(keyStoreType);
- InputStream in = null;
- try {
- in = getFileInputStream(keyStoreFile);
- if (in == null) {
- LOG.error("Unable to obtain
keystore from file [" + keyStoreFile + "]");
- return;
- }
- keyStore.load(in,
keyStoreFilepwd.toCharArray());
- KeyManagerFactory keyManagerFactory =
KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
- keyManagerFactory.init(keyStore,
keyStoreFilepwd.toCharArray());
- kmList =
keyManagerFactory.getKeyManagers();
- }
- finally {
- if (in != null) {
- in.close();
- }
- }
-
- }
-
- if (trustStoreFile != null && trustStoreFilepwd !=
null) {
-
- KeyStore trustStore =
KeyStore.getInstance(trustStoreType);
- InputStream in = null;
- try {
- in = getFileInputStream(trustStoreFile);
- if (in == null) {
- LOG.error("Unable to obtain
keystore from file [" + trustStoreFile + "]");
- return;
- }
- trustStore.load(in,
trustStoreFilepwd.toCharArray());
- TrustManagerFactory trustManagerFactory
= TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
- trustManagerFactory.init(trustStore);
- tmList =
trustManagerFactory.getTrustManagers();
- }
- finally {
- if (in != null) {
- in.close();
- }
- }
- }
-
- sslContext = SSLContext.getInstance("TLSv1.2");
-
- sslContext.init(kmList, tmList, new SecureRandom());
- sockFactory = sslContext.getSocketFactory();
- }
- catch(Throwable t) {
- throw new RuntimeException("Unable to create
SSLConext for communication to policy manager", t);
- }
+ SSLContext sslContext = null;
+ String keyStoreFile = config.getSSLKeyStorePath();
+ String keyStoreFilepwd = config.getSSLKeyStorePathPassword();
+ String trustStoreFile = config.getSSLTrustStorePath();
+ String trustStoreFilepwd = config.getSSLTrustStorePathPassword();
+ String keyStoreType = config.getSSLKeyStoreType();
+ String trustStoreType = config.getSSLTrustStoreType();
+ try {
+ KeyManager[] kmList = null;
+ TrustManager[] tmList = null;
+
+ if (keyStoreFile != null && keyStoreFilepwd != null) {
+ KeyStore keyStore = KeyStore.getInstance(keyStoreType);
+ InputStream in = null;
+ try {
Review Comment:
Consider replacing finally with try-with-resource: `try (InputStream in
getFileInputStream(keyStoreFile)) {`.
Same at line 85 as well.
##########
ugsync/src/main/java/org/apache/ranger/ldapusersync/process/CustomSSLSocketFactory.java:
##########
@@ -30,89 +42,70 @@
import java.security.KeyStore;
import java.security.SecureRandom;
-import javax.net.ssl.KeyManager;
-import javax.net.ssl.KeyManagerFactory;
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.SSLSocketFactory;
-import javax.net.ssl.TrustManager;
-import javax.net.ssl.TrustManagerFactory;
-
-import org.apache.ranger.unixusersync.config.UserGroupSyncConfig;
-import org.apache.ranger.unixusersync.process.PolicyMgrUserGroupBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class CustomSSLSocketFactory extends SSLSocketFactory{
- private static final Logger LOG =
LoggerFactory.getLogger(CustomSSLSocketFactory.class);
- private SSLSocketFactory sockFactory;
- private UserGroupSyncConfig config = UserGroupSyncConfig.getInstance();
+public class CustomSSLSocketFactory extends SSLSocketFactory {
+ private static final Logger LOG =
LoggerFactory.getLogger(CustomSSLSocketFactory.class);
+ private SSLSocketFactory sockFactory;
Review Comment:
I suggest separating static and non-static members with a blank line.
##########
ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java:
##########
@@ -52,1240 +50,1161 @@
import javax.naming.ldap.StartTlsRequest;
import javax.naming.ldap.StartTlsResponse;
-import org.apache.commons.collections.MapUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.thirdparty.com.google.common.collect.HashBasedTable;
-import org.apache.hadoop.thirdparty.com.google.common.collect.Table;
-import org.apache.ranger.ugsyncutil.util.UgsyncCommonConstants;
-import org.apache.ranger.unixusersync.config.UserGroupSyncConfig;
-import org.apache.ranger.ugsyncutil.model.LdapSyncSourceInfo;
-import org.apache.ranger.ugsyncutil.model.UgsyncAuditInfo;
-import org.apache.ranger.usergroupsync.UserGroupSink;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.ranger.usergroupsync.UserGroupSource;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Properties;
+import java.util.Set;
+import java.util.UUID;
public class LdapUserGroupBuilder implements UserGroupSource {
-
- private static final Logger LOG =
LoggerFactory.getLogger(LdapUserGroupBuilder.class);
-
- private UserGroupSyncConfig config = UserGroupSyncConfig.getInstance();
-
- private static final String DATA_TYPE_BYTEARRAY = "byte[]";
- private static final String DATE_FORMAT = "yyyyMMddHHmmss";
- private static final int PAGE_SIZE = 500;
- private static final String MEMBER_OF_ATTR = "memberof=";
- private static final String GROUP_NAME_ATTRIBUTE = "cn=";
- private static long deltaSyncUserTime = 0; // Used for AD uSNChanged
- private static long deltaSyncGroupTime = 0; // Used for AD uSNChanged
- private String deltaSyncUserTimeStamp; // Used for OpenLdap
modifyTimestamp
- private String deltaSyncGroupTimeStamp; // Used for OpenLdap
modifyTimestamp
-
- private String ldapUrl;
- private String ldapBindDn;
- private String ldapBindPassword;
- private String ldapAuthenticationMechanism;
- private String ldapReferral;
- private String searchBase;
-
- private String[] userSearchBase;
- private String userNameAttribute;
- private String userCloudIdAttribute;
- private int userSearchScope;
- private String userObjectClass;
- private String userSearchFilter;
- private Set<String> groupNameSet;
- private String extendedUserSearchFilter;
- private SearchControls userSearchControls;
- private Set<String> userGroupNameAttributeSet;
- private Set<String> otherUserAttributes;
-
- private boolean pagedResultsEnabled = true;
- private int pagedResultsSize = PAGE_SIZE;
-
- private boolean groupSearchFirstEnabled = true;
- private boolean userSearchEnabled = true;
- private boolean groupSearchEnabled = true;
- private String[] groupSearchBase;
- private int groupSearchScope;
- private String groupObjectClass;
- private String groupSearchFilter;
- private String extendedGroupSearchFilter;
- private String extendedAllGroupsSearchFilter;
- private SearchControls groupSearchControls;
- private String groupMemberAttributeName;
- private String groupNameAttribute;
- private String groupCloudIdAttribute;
- private Set<String> otherGroupAttributes;
- private int groupHierarchyLevels;
- private int deleteCycles;
- private String currentSyncSource;
-
- private LdapContext ldapContext;
- StartTlsResponse tls;
-
- private Table<String, String, String> groupUserTable;
- UgsyncAuditInfo ugsyncAuditInfo;
- LdapSyncSourceInfo ldapSyncSourceInfo;
-
- private Map<String, Map<String, String>> sourceUsers; // key is user DN
and value is map of user attributes containing original name, DN, etc...
- private Map<String, Map<String, String>> sourceGroups; // key is group
DN and value is map of group attributes containing original name, DN, etc...
- private Map<String, Set<String>> sourceGroupUsers; // key is group DN
and value is set of user DNs (members)
-
- public static void main(String[] args) throws Throwable {
- LdapUserGroupBuilder ugBuilder = new LdapUserGroupBuilder();
- ugBuilder.init();
- }
-
- @Override
- public void init() throws Throwable{
- deltaSyncUserTime = 0;
- deltaSyncGroupTime = 0;
- deleteCycles = 1;
- DateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT);
- deltaSyncUserTimeStamp = dateFormat.format(new Date(0));
- deltaSyncGroupTimeStamp = dateFormat.format(new Date(0));
- setConfig();
- ugsyncAuditInfo = new UgsyncAuditInfo();
- ldapSyncSourceInfo = new LdapSyncSourceInfo();
- ldapSyncSourceInfo.setLdapUrl(ldapUrl);
-
ldapSyncSourceInfo.setIncrementalSycn(Boolean.toString(config.isDeltaSyncEnabled()));
-
ldapSyncSourceInfo.setUserSearchEnabled(Boolean.toString(userSearchEnabled));
-
ldapSyncSourceInfo.setGroupSearchEnabled(Boolean.toString(groupSearchEnabled));
-
ldapSyncSourceInfo.setGroupSearchFirstEnabled(Boolean.toString(groupSearchFirstEnabled));
-
ldapSyncSourceInfo.setGroupHierarchyLevel(Integer.toString(groupHierarchyLevels));
- ugsyncAuditInfo.setSyncSource(currentSyncSource);
- ugsyncAuditInfo.setLdapSyncSourceInfo(ldapSyncSourceInfo);
- }
-
- private void createLdapContext() throws Throwable {
- Properties env = new Properties();
- env.put(Context.INITIAL_CONTEXT_FACTORY,
- "com.sun.jndi.ldap.LdapCtxFactory");
- env.put(Context.PROVIDER_URL, ldapUrl);
- if (ldapUrl.startsWith("ldaps") &&
(config.getSSLTrustStorePath() != null &&
!config.getSSLTrustStorePath().trim().isEmpty())) {
- env.put("java.naming.ldap.factory.socket",
"org.apache.ranger.ldapusersync.process.CustomSSLSocketFactory");
- }
-
- if (StringUtils.isNotEmpty(userCloudIdAttribute)) {
- if
(config.getUserCloudIdAttributeDataType().equals(DATA_TYPE_BYTEARRAY)) {
- env.put("java.naming.ldap.attributes.binary",
userCloudIdAttribute);
- }
- }
-
- if (StringUtils.isNotEmpty(groupCloudIdAttribute)) {
- if
(config.getGroupCloudIdAttributeDataType().equals(DATA_TYPE_BYTEARRAY)) {
- env.put("java.naming.ldap.attributes.binary",
groupCloudIdAttribute);
- }
- }
-
- for (String otherUserAttribute : otherUserAttributes) {
- String attrType =
config.getOtherUserAttributeDataType(otherUserAttribute);
- if (attrType.equals(DATA_TYPE_BYTEARRAY)) {
- env.put("java.naming.ldap.attributes.binary",
otherUserAttribute);
- }
- }
-
- for (String otherGroupAttribute : otherGroupAttributes) {
- String attrType =
config.getOtherGroupAttributeDataType(otherGroupAttribute);
- if (attrType.equals(DATA_TYPE_BYTEARRAY)) {
- env.put("java.naming.ldap.attributes.binary",
otherGroupAttribute);
- }
- }
- ldapContext = new InitialLdapContext(env, null);
- if (!ldapUrl.startsWith("ldaps")) {
- if (config.isStartTlsEnabled()) {
- tls = (StartTlsResponse)
ldapContext.extendedOperation(new StartTlsRequest());
- if (config.getSSLTrustStorePath() != null &&
!config.getSSLTrustStorePath().trim().isEmpty()) {
-
tls.negotiate(CustomSSLSocketFactory.getDefault());
- } else {
- tls.negotiate();
- }
- LOG.info("Starting TLS session...");
- }
- }
-
- ldapContext.addToEnvironment(Context.SECURITY_PRINCIPAL,
ldapBindDn);
- ldapContext.addToEnvironment(Context.SECURITY_CREDENTIALS,
ldapBindPassword);
- ldapContext.addToEnvironment(Context.SECURITY_AUTHENTICATION,
ldapAuthenticationMechanism);
- ldapContext.addToEnvironment(Context.REFERRAL, ldapReferral);
- }
-
- private void setConfig() throws Throwable {
- LOG.info("LdapUserGroupBuilder initialization started");
-
- currentSyncSource = config.getCurrentSyncSource();
- groupSearchFirstEnabled = true;
- userSearchEnabled = config.isUserSearchEnabled();
- groupSearchEnabled = config.isGroupSearchEnabled();
- ldapUrl = config.getLdapUrl();
- ldapBindDn = config.getLdapBindDn();
- ldapBindPassword = config.getLdapBindPassword();
- //ldapBindPassword = "admin-password";
- ldapAuthenticationMechanism =
config.getLdapAuthenticationMechanism();
- ldapReferral = config.getContextReferral();
- searchBase = config.getSearchBase();
-
- userSearchBase = config.getUserSearchBase().split(";");
- userSearchScope = config.getUserSearchScope();
- userObjectClass = config.getUserObjectClass();
- userSearchFilter = config.getUserSearchFilter();
- userNameAttribute = config.getUserNameAttribute();
- userCloudIdAttribute = config.getUserCloudIdAttribute();
-
- Set<String> userSearchAttributes = new HashSet<String>();
- userSearchAttributes.add(userNameAttribute);
- userGroupNameAttributeSet =
config.getUserGroupNameAttributeSet();
- for (String useGroupNameAttribute : userGroupNameAttributeSet) {
- userSearchAttributes.add(useGroupNameAttribute);
- }
- userSearchAttributes.add(userCloudIdAttribute);
- otherUserAttributes = config.getOtherUserAttributes();
- for (String otherUserAttribute : otherUserAttributes) {
- userSearchAttributes.add(otherUserAttribute);
- }
- userSearchAttributes.add("uSNChanged");
- userSearchAttributes.add("modifytimestamp");
- userSearchControls = new SearchControls();
- userSearchControls.setSearchScope(userSearchScope);
-
userSearchControls.setReturningAttributes(userSearchAttributes.toArray(
- new String[userSearchAttributes.size()]));
-
- pagedResultsEnabled = config.isPagedResultsEnabled();
- pagedResultsSize = config.getPagedResultsSize();
-
- groupSearchBase = config.getGroupSearchBase().split(";");
- groupSearchScope = config.getGroupSearchScope();
- groupObjectClass = config.getGroupObjectClass();
- groupSearchFilter = config.getGroupSearchFilter();
- groupMemberAttributeName =
config.getUserGroupMemberAttributeName();
- groupNameAttribute = config.getGroupNameAttribute();
- groupCloudIdAttribute = config.getGroupCloudIdAttribute();
- groupHierarchyLevels = config.getGroupHierarchyLevels();
-
- extendedGroupSearchFilter = "(&" + extendedGroupSearchFilter +
"(|(" + groupMemberAttributeName + "={0})(" + groupMemberAttributeName +
"={1})))";
-
- groupSearchControls = new SearchControls();
- groupSearchControls.setSearchScope(groupSearchScope);
-
- Set<String> groupSearchAttributes = new HashSet<String>();
- groupSearchAttributes.add(groupNameAttribute);
- groupSearchAttributes.add(groupCloudIdAttribute);
- groupSearchAttributes.add(groupMemberAttributeName);
- groupSearchAttributes.add("uSNChanged");
- groupSearchAttributes.add("modifytimestamp");
- otherGroupAttributes = config.getOtherGroupAttributes();
- for (String otherGroupAttribute : otherGroupAttributes) {
- groupSearchAttributes.add(otherGroupAttribute);
- }
-
groupSearchControls.setReturningAttributes(groupSearchAttributes.toArray(
- new String[groupSearchAttributes.size()]));
-
- if (StringUtils.isEmpty(userSearchFilter)) {
- groupNameSet = config.getGroupNameSet();
- String computedSearchFilter = "";
- for (String groupName : groupNameSet) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("groupName = " + groupName);
- }
- if (!groupName.startsWith(MEMBER_OF_ATTR) &&
!groupName.startsWith(GROUP_NAME_ATTRIBUTE)){
- LOG.info("Ignoring unsupported format
for " + groupName );
- continue;
- }
- String searchFilter = groupName;
- if (groupName.startsWith(MEMBER_OF_ATTR)) {
- searchFilter =
groupName.substring(MEMBER_OF_ATTR.length());
- }
- searchFilter = getFirstRDN(searchFilter);
- computedSearchFilter +=
getDNForMemberOf(searchFilter);
- }
- if (StringUtils.isNotEmpty(computedSearchFilter)) {
- computedSearchFilter = "(|" +
computedSearchFilter + ")";
- }
- LOG.info("Final computedSearchFilter = " +
computedSearchFilter);
- userSearchFilter = computedSearchFilter;
- }
-
- LOG.info("LdapUserGroupBuilder initialization completed with --
"
- + "ldapUrl: " + ldapUrl
- + ", ldapBindDn: " + ldapBindDn
- + ", ldapBindPassword: ***** "
- + ", ldapAuthenticationMechanism: " +
ldapAuthenticationMechanism
- + ", searchBase: " + searchBase
- + ", userSearchBase: " +
Arrays.toString(userSearchBase)
- + ", userSearchScope: " + userSearchScope
- + ", userObjectClass: " + userObjectClass
- + ", userSearchFilter: " + userSearchFilter
- + ", extendedUserSearchFilter: " +
extendedUserSearchFilter
- + ", userNameAttribute: " + userNameAttribute
- + ", userSearchAttributes: " +
userSearchAttributes
- + ", userGroupNameAttributeSet: " +
userGroupNameAttributeSet
- + ", otherUserAttributes: " +
otherUserAttributes
- + ", pagedResultsEnabled: " +
pagedResultsEnabled
- + ", pagedResultsSize: " + pagedResultsSize
- + ", groupSearchEnabled: " + groupSearchEnabled
- + ", groupSearchBase: " +
Arrays.toString(groupSearchBase)
- + ", groupSearchScope: " + groupSearchScope
- + ", groupObjectClass: " + groupObjectClass
- + ", groupSearchFilter: " + groupSearchFilter
- + ", extendedGroupSearchFilter: " +
extendedGroupSearchFilter
- + ", extendedAllGroupsSearchFilter: " +
extendedAllGroupsSearchFilter
- + ", groupMemberAttributeName: " +
groupMemberAttributeName
- + ", groupNameAttribute: " + groupNameAttribute
- + ", groupSearchAttributes: " +
groupSearchAttributes
- + ", groupSearchFirstEnabled: " +
groupSearchFirstEnabled
- + ", userSearchEnabled: " + userSearchEnabled
- + ", ldapReferral: " + ldapReferral
- );
- }
-
- private void closeLdapContext() throws Throwable {
- if (tls != null) {
- tls.close();
- }
- if (ldapContext != null) {
- ldapContext.close();
- }
- }
-
- @Override
- public boolean isChanged() {
- // we do not want to get the full ldap dit and check whether
anything has changed
- return true;
- }
-
- @Override
- public void updateSink(UserGroupSink sink) throws Throwable {
- LOG.info("LdapUserGroupBuilder updateSink started");
+ private static final Logger LOG =
LoggerFactory.getLogger(LdapUserGroupBuilder.class);
+ private static final String DATA_TYPE_BYTEARRAY = "byte[]";
+ private static final String DATE_FORMAT = "yyyyMMddHHmmss";
+ private static final String MEMBER_OF_ATTR = "memberof=";
+ private static final String GROUP_NAME_ATTRIBUTE = "cn=";
+ private static final int PAGE_SIZE = 500;
+ /* for AD uSNChanged */
+ private static long deltaSyncUserTime;
+ private static long deltaSyncGroupTime;
+ /* ***************** */
+ private boolean pagedResultsEnabled = true;
+ private boolean groupSearchFirstEnabled = true;
+ private boolean userSearchEnabled = true;
+ private boolean groupSearchEnabled = true;
+ private int pagedResultsSize = PAGE_SIZE;
+ private int groupHierarchyLevels;
+ private int deleteCycles;
+ private int userSearchScope;
+ private int groupSearchScope;
+
+ private UserGroupSyncConfig config = UserGroupSyncConfig.getInstance();
+ /* for OpenLdap modifyTimestamp */
+ private String deltaSyncUserTimeStamp;
+ private String deltaSyncGroupTimeStamp;
+ /* ******************************** */
+ private String ldapUrl;
+ private String ldapBindDn;
+ private String ldapBindPassword;
+ private String ldapAuthenticationMechanism;
+ private String ldapReferral;
+ private String searchBase;
+ private String userNameAttribute;
+ private String userCloudIdAttribute;
+ private String userObjectClass;
+ private String userSearchFilter;
+ private String extendedUserSearchFilter;
+ private String groupObjectClass;
+ private String groupSearchFilter;
+ private String extendedGroupSearchFilter;
+ private String extendedAllGroupsSearchFilter;
+ private String groupMemberAttributeName;
+ private String groupNameAttribute;
+ private String groupCloudIdAttribute;
+ private String currentSyncSource;
+ private String[] userSearchBase;
+ private String[] groupSearchBase;
+ private Set<String> groupNameSet;
+ private Set<String> userGroupNameAttributeSet;
+ private Set<String> otherUserAttributes;
+ private Set<String> otherGroupAttributes;
+ private LdapContext ldapContext;
+ private SearchControls userSearchControls;
+ private SearchControls groupSearchControls;
+ private Table<String, String, String> groupUserTable;
+ /* { key = user DN, value = map of user attributes {original name, DN,
etc.}} */
+ private Map<String, Map<String, String>> sourceUsers;
+ /* { key = group DN, value = map of group attributes {original name, DN,
etc.}} */
+ private Map<String, Map<String, String>> sourceGroups;
+ /* { key = group DN, value = set of user DNs (members) } */
+ private Map<String, Set<String>> sourceGroupUsers;
+
+ StartTlsResponse tls;
+ UgsyncAuditInfo ugsyncAuditInfo;
+ LdapSyncSourceInfo ldapSyncSourceInfo;
+
+ public static void main(String[] args) throws Throwable {
+ LdapUserGroupBuilder ugBuilder = new LdapUserGroupBuilder();
+ ugBuilder.init();
+ }
+
+ @Override
+ public void init() throws Throwable {
+ deltaSyncUserTime = 0;
+ deltaSyncGroupTime = 0;
+ deleteCycles = 1;
+ DateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT);
+ deltaSyncUserTimeStamp = dateFormat.format(new Date(0));
+ deltaSyncGroupTimeStamp = dateFormat.format(new Date(0));
+ setConfig();
+ ugsyncAuditInfo = new UgsyncAuditInfo();
+ ldapSyncSourceInfo = new LdapSyncSourceInfo();
+ ldapSyncSourceInfo.setLdapUrl(ldapUrl);
+
ldapSyncSourceInfo.setIncrementalSycn(Boolean.toString(config.isDeltaSyncEnabled()));
+
ldapSyncSourceInfo.setUserSearchEnabled(Boolean.toString(userSearchEnabled));
+
ldapSyncSourceInfo.setGroupSearchEnabled(Boolean.toString(groupSearchEnabled));
+
ldapSyncSourceInfo.setGroupSearchFirstEnabled(Boolean.toString(groupSearchFirstEnabled));
+
ldapSyncSourceInfo.setGroupHierarchyLevel(Integer.toString(groupHierarchyLevels));
+ ugsyncAuditInfo.setSyncSource(currentSyncSource);
+ ugsyncAuditInfo.setLdapSyncSourceInfo(ldapSyncSourceInfo);
+ }
+
+ @Override
+ public boolean isChanged() {
+ // we do not want to get the full ldap dit and check whether anything
has changed
+ return true;
+ }
+
+ @Override
+ public void updateSink(UserGroupSink sink) throws Throwable {
+ LOG.info("LdapUserGroupBuilder updateSink started");
boolean computeDeletes = false;
- groupUserTable = HashBasedTable.create();
- sourceGroups = new HashMap<>();
- sourceUsers = new HashMap<>();
- sourceGroupUsers = new HashMap<>();
- long highestdeltaSyncUserTime = 0;
- long highestdeltaSyncGroupTime = 0;
-
- if (config.isUserSyncDeletesEnabled() && deleteCycles >=
config.getUserSyncDeletesFrequency()) {
- deleteCycles = 1;
- computeDeletes = true;
- if (LOG.isDebugEnabled()) {
- LOG.debug("Compute deleted users/groups is
enabled for this sync cycle");
- }
- }
- if (config.isUserSyncDeletesEnabled()) {
- deleteCycles++;
- }
- if (groupSearchEnabled) {
- highestdeltaSyncGroupTime = getGroups(computeDeletes);
- }
- if (userSearchEnabled) {
- LOG.info("Performing user search to retrieve users from
AD/LDAP");
- highestdeltaSyncUserTime = getUsers(computeDeletes);
- }
-
- if (groupHierarchyLevels > 0) {
- LOG.info("Going through group hierarchy for nested
group evaluation");
+ groupUserTable = HashBasedTable.create();
+ sourceGroups = new HashMap<>();
+ sourceUsers = new HashMap<>();
+ sourceGroupUsers = new HashMap<>();
+ long highestdeltaSyncUserTime = 0;
+ long highestdeltaSyncGroupTime = 0;
+
+ if (config.isUserSyncDeletesEnabled() && deleteCycles >=
config.getUserSyncDeletesFrequency()) {
+ deleteCycles = 1;
+ computeDeletes = true;
+ LOG.debug("Compute deleted users/groups is enabled for this sync
cycle");
+ }
+ if (config.isUserSyncDeletesEnabled()) {
+ deleteCycles++;
+ }
+ if (groupSearchEnabled) {
+ highestdeltaSyncGroupTime = getGroups(computeDeletes);
+ }
+ if (userSearchEnabled) {
+ LOG.info("Performing user search to retrieve users from AD/LDAP");
+ highestdeltaSyncUserTime = getUsers(computeDeletes);
+ }
+
+ if (groupHierarchyLevels > 0) {
+ LOG.info("Going through group hierarchy for nested group
evaluation");
Set<String> groupFullNames = sourceGroups.keySet();
- for(String group : groupFullNames) {
- Set<String> nextLevelGroups =
groupUserTable.column(group).keySet();
- goUpGroupHierarchy(nextLevelGroups,
groupHierarchyLevels-1, group);
- }
- LOG.info("Completed group hierarchy computation");
- }
-
- Iterator<String> groupUserTableIterator =
groupUserTable.rowKeySet().iterator();
- while (groupUserTableIterator.hasNext()) {
- String groupName = groupUserTableIterator.next();
- Map<String,String> groupUsersMap =
groupUserTable.row(groupName);
- Set<String> userSet = new HashSet<String>();
- for(Map.Entry<String, String> entry :
groupUsersMap.entrySet()){
- if (sourceUsers.containsKey(entry.getValue())) {
- userSet.add(entry.getValue());
- }
- }
- sourceGroupUsers.put(groupName, userSet);
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Users = " + sourceUsers.keySet());
- LOG.debug("Groups = " + sourceGroups.keySet());
- LOG.debug("GroupUsers = " + sourceGroupUsers.keySet());
- }
-
- try {
- sink.addOrUpdateUsersGroups(sourceGroups, sourceUsers,
sourceGroupUsers, computeDeletes);
- DateFormat dateFormat = new
SimpleDateFormat(DATE_FORMAT);
- LOG.info("deltaSyncUserTime = " + deltaSyncUserTime + "
and highestdeltaSyncUserTime = " + highestdeltaSyncUserTime);
- if (deltaSyncUserTime < highestdeltaSyncUserTime) {
- // Incrementing highestdeltaSyncUserTime (for
AD) in order to avoid search record repetition for next sync cycle.
- deltaSyncUserTime = highestdeltaSyncUserTime +
1;
- // Incrementing the highest timestamp value
(for Openldap) with 1sec in order to avoid search record repetition for next
sync cycle.
- deltaSyncUserTimeStamp = dateFormat.format(new
Date(highestdeltaSyncUserTime + 60l));
- }
-
- LOG.info("deltaSyncGroupTime = " + deltaSyncGroupTime +
" and highestdeltaSyncGroupTime = " + highestdeltaSyncGroupTime);
- // Update deltaSyncUserTime/deltaSyncUserTimeStamp here
so that in case of failures, we get updates in next cycle
- if (deltaSyncGroupTime < highestdeltaSyncGroupTime) {
- // Incrementing highestdeltaSyncGroupTime (for
AD) in order to avoid search record repetition for next sync cycle.
- deltaSyncGroupTime =
highestdeltaSyncGroupTime+1;
- // Incrementing the highest timestamp value
(for OpenLdap) with 1min in order to avoid search record repetition for next
sync cycle.
- deltaSyncGroupTimeStamp = dateFormat.format(new
Date(highestdeltaSyncGroupTime + 60l));
- }
- } catch (Throwable t) {
- LOG.error("Failed to update ranger admin. Will retry in
next sync cycle!!", t);
- }
-
-
ldapSyncSourceInfo.setUserSearchFilter(extendedUserSearchFilter);
-
ldapSyncSourceInfo.setGroupSearchFilter(extendedAllGroupsSearchFilter);
-
- try {
- sink.postUserGroupAuditInfo(ugsyncAuditInfo);
- } catch (Throwable t) {
- LOG.error("sink.postUserGroupAuditInfo failed with
exception: " + t.getMessage());
- }
- }
-
- private long getUsers(boolean computeDeletes) throws Throwable {
- NamingEnumeration<SearchResult> userSearchResultEnum = null;
- NamingEnumeration<SearchResult> groupSearchResultEnum = null;
- long highestdeltaSyncUserTime;
- try {
- createLdapContext();
- int total;
- // Activate paged results
- if (pagedResultsEnabled) {
- ldapContext.setRequestControls(new Control[]{
- new
PagedResultsControl(pagedResultsSize, Control.NONCRITICAL) });
- }
- DateFormat dateFormat = new
SimpleDateFormat(DATE_FORMAT);
- if (groupUserTable.rowKeySet().size() != 0 ||
!config.isDeltaSyncEnabled() || (computeDeletes)) {
- // Fix RANGER-1957: Perform full sync when
there are updates to the groups or when incremental sync is not enabled
- deltaSyncUserTime = 0;
- deltaSyncUserTimeStamp = dateFormat.format(new
Date(0));
- }
-
- extendedUserSearchFilter = "(objectclass=" +
userObjectClass + ")(|(uSNChanged>=" + deltaSyncUserTime +
")(modifyTimestamp>=" + deltaSyncUserTimeStamp + "Z))";
-
- if (userSearchFilter != null &&
!userSearchFilter.trim().isEmpty()) {
- String customFilter = userSearchFilter.trim();
- if (!customFilter.startsWith("(")) {
- customFilter = "(" + customFilter + ")";
- }
-
- extendedUserSearchFilter = "(&" +
extendedUserSearchFilter + customFilter + ")";
- } else {
- extendedUserSearchFilter = "(&" +
extendedUserSearchFilter + ")";
- }
- LOG.info("extendedUserSearchFilter = " +
extendedUserSearchFilter);
-
- highestdeltaSyncUserTime = deltaSyncUserTime;
-
- // When multiple OUs are configured, go through each OU
as the user search base to search for users.
- for (int ou=0; ou<userSearchBase.length; ou++) {
- byte[] cookie = null;
- int counter = 0;
- try {
- int paged = 0;
- do {
- userSearchResultEnum = ldapContext
-
.search(userSearchBase[ou], extendedUserSearchFilter,
-
userSearchControls);
-
- while (userSearchResultEnum.hasMore()) {
- // searchResults contains all
the user entries
- final SearchResult userEntry =
userSearchResultEnum.next();
-
- if (userEntry == null) {
- LOG.info("userEntry
null, skipping sync for the entry");
- continue;
- }
-
- Attributes attributes =
userEntry.getAttributes();
- if (attributes == null) {
- LOG.info("attributes
missing for entry " + userEntry.getNameInNamespace() +
- ",
skipping sync");
- continue;
- }
-
- Attribute userNameAttr =
attributes.get(userNameAttribute);
- if (userNameAttr == null) {
-
LOG.info(userNameAttribute + " missing for entry " +
userEntry.getNameInNamespace() +
- ",
skipping sync");
- continue;
- }
-
- String userFullName =
(userEntry.getNameInNamespace());
- String userName = (String)
userNameAttr.get();
-
- if (userName == null ||
userName.trim().isEmpty()) {
-
LOG.info(userNameAttribute + " empty for entry " +
userEntry.getNameInNamespace() +
- ",
skipping sync");
- continue;
- }
-
- Attribute timeStampAttr =
attributes.get("uSNChanged");
- if (timeStampAttr != null) {
- String uSNChangedVal =
(String) timeStampAttr.get();
- long
currentDeltaSyncTime = Long.parseLong(uSNChangedVal);
- LOG.info("uSNChangedVal
= " + uSNChangedVal + "and currentDeltaSyncTime = " + currentDeltaSyncTime);
- if
(currentDeltaSyncTime > highestdeltaSyncUserTime) {
-
highestdeltaSyncUserTime = currentDeltaSyncTime;
- }
- } else {
- timeStampAttr =
attributes.get("modifytimestamp");
- if (timeStampAttr !=
null) {
- String
timeStampVal = (String) timeStampAttr.get();
- Date parseDate
= dateFormat.parse(timeStampVal);
- long
currentDeltaSyncTime = parseDate.getTime();
-
LOG.info("timeStampVal = " + timeStampVal + "and currentDeltaSyncTime = " +
currentDeltaSyncTime);
- if
(currentDeltaSyncTime > highestdeltaSyncUserTime) {
-
highestdeltaSyncUserTime = currentDeltaSyncTime;
-
deltaSyncUserTimeStamp = timeStampVal;
- }
- }
- }
-
- // Get all the groups from the
group name attribute of the user only when group search is not enabled.
- if (!groupSearchEnabled) {
- for (String
useGroupNameAttribute : userGroupNameAttributeSet) {
- Attribute
userGroupfAttribute = userEntry.getAttributes().get(useGroupNameAttribute);
- if
(userGroupfAttribute != null) {
-
NamingEnumeration<?> groupEnum = userGroupfAttribute.getAll();
- while
(groupEnum.hasMore()) {
-
String groupDN = (String) groupEnum.next();
-
if (LOG.isDebugEnabled()) {
-
LOG.debug("Adding " + groupDN + " to " + userName);
-
}
-
Map<String, String> groupAttrMap = new HashMap<>();
-
String groupName = getShortName(groupDN);
-
groupAttrMap.put(UgsyncCommonConstants.ORIGINAL_NAME, groupName);
-
groupAttrMap.put(UgsyncCommonConstants.FULL_NAME, groupDN);
-
groupAttrMap.put(UgsyncCommonConstants.SYNC_SOURCE, currentSyncSource);
-
groupAttrMap.put(UgsyncCommonConstants.LDAP_URL, config.getLdapUrl());
-
sourceGroups.put(groupDN, groupAttrMap);
-
if (LOG.isDebugEnabled()) {
-
LOG.debug("As groupsearch is disabled, adding group " + groupName + "
from user memberof attribute for user " + userName);
-
}
-
groupUserTable.put(groupDN, userFullName, userFullName);
- }
- }
- }
- }
-
- Map<String, String> userAttrMap
= new HashMap<>();
-
userAttrMap.put(UgsyncCommonConstants.ORIGINAL_NAME, userName);
-
userAttrMap.put(UgsyncCommonConstants.FULL_NAME, userFullName);
-
userAttrMap.put(UgsyncCommonConstants.SYNC_SOURCE, currentSyncSource);
-
userAttrMap.put(UgsyncCommonConstants.LDAP_URL, config.getLdapUrl());
- Attribute userCloudIdAttr =
attributes.get(userCloudIdAttribute);
- if (userCloudIdAttr != null) {
-
addToAttrMap(userAttrMap, "cloud_id", userCloudIdAttr,
config.getUserCloudIdAttributeDataType());
- }
- for (String otherUserAttribute
: otherUserAttributes) {
- if
(attributes.get(otherUserAttribute) != null) {
- String attrType
= config.getOtherUserAttributeDataType(otherUserAttribute);
-
addToAttrMap(userAttrMap, otherUserAttribute,
attributes.get(otherUserAttribute), attrType);
- }
- }
-
- sourceUsers.put(userFullName,
userAttrMap);
- if
((groupUserTable.containsColumn(userFullName) ||
groupUserTable.containsColumn(userName))) {
- //Update the username
in the groupUserTable with the one from username attribute.
- Map<String, String>
userMap = groupUserTable.column(userFullName);
- if
(MapUtils.isEmpty(userMap)) {
- userMap =
groupUserTable.column(userName);
- }
- for (Map.Entry<String,
String> entry : userMap.entrySet()) {
- if
(LOG.isDebugEnabled()) {
-
LOG.debug("Updating groupUserTable " + entry.getValue() + " with: " + userName
+ " for " + entry.getKey());
- }
-
groupUserTable.put(entry.getKey(), userFullName, userFullName);
- }
- }
- counter++;
-
- if (counter <= 2000) {
- LOG.info("Updating user count: " + counter + ",
userName: " + userName);
- if ( counter == 2000 ) {
- LOG.info("===> 2000 user records have been
synchronized so far. From now on, only a summary progress log will be written
for every 100 users. To continue to see detailed log for every user, please
enable Trace level logging. <===");
+ for (String group : groupFullNames) {
+ Set<String> nextLevelGroups =
groupUserTable.column(group).keySet();
+ goUpGroupHierarchy(nextLevelGroups, groupHierarchyLevels - 1,
group);
+ }
+ LOG.info("Completed group hierarchy computation");
+ }
+
+ Iterator<String> groupUserTableIterator =
groupUserTable.rowKeySet().iterator();
+ while (groupUserTableIterator.hasNext()) {
+ String groupName = groupUserTableIterator.next();
+ Map<String, String> groupUsersMap = groupUserTable.row(groupName);
+ Set<String> userSet = new HashSet<String>();
+ for (Map.Entry<String, String> entry : groupUsersMap.entrySet()) {
+ if (sourceUsers.containsKey(entry.getValue())) {
+ userSet.add(entry.getValue());
+ }
+ }
+ sourceGroupUsers.put(groupName, userSet);
+ }
+
+ LOG.debug("Users = {}", sourceUsers.keySet());
+ LOG.debug("Groups = {}", sourceGroups.keySet());
+ LOG.debug("GroupUsers = {}", sourceGroupUsers.keySet());
+
+ try {
+ sink.addOrUpdateUsersGroups(sourceGroups, sourceUsers,
sourceGroupUsers, computeDeletes);
+ DateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT);
+ LOG.info("deltaSyncUserTime = {} and highestdeltaSyncUserTime =
{}", deltaSyncUserTime, highestdeltaSyncUserTime);
+ if (deltaSyncUserTime < highestdeltaSyncUserTime) {
+ // Incrementing highestdeltaSyncUserTime (for AD) in order to
avoid search record repetition for next sync cycle.
+ deltaSyncUserTime = highestdeltaSyncUserTime + 1;
+ // Incrementing the highest timestamp value (for Openldap)
with 1sec in order to avoid search record repetition for next sync cycle.
+ deltaSyncUserTimeStamp = dateFormat.format(new
Date(highestdeltaSyncUserTime + 60L));
+ }
+
+ LOG.info("deltaSyncGroupTime = {} and highestdeltaSyncGroupTime =
{} ", deltaSyncGroupTime, highestdeltaSyncGroupTime);
+ // Update deltaSyncUserTime/deltaSyncUserTimeStamp here so that in
case of failures, we get updates in next cycle
+ if (deltaSyncGroupTime < highestdeltaSyncGroupTime) {
+ // Incrementing highestdeltaSyncGroupTime (for AD) in order to
avoid search record repetition for next sync cycle.
+ deltaSyncGroupTime = highestdeltaSyncGroupTime + 1;
+ // Incrementing the highest timestamp value (for OpenLdap)
with 1min in order to avoid search record repetition for next sync cycle.
+ deltaSyncGroupTimeStamp = dateFormat.format(new
Date(highestdeltaSyncGroupTime + 60L));
+ }
+ } catch (Throwable t) {
+ LOG.error("Failed to update ranger admin. Will retry in next sync
cycle!!", t);
+ }
+
+ ldapSyncSourceInfo.setUserSearchFilter(extendedUserSearchFilter);
+ ldapSyncSourceInfo.setGroupSearchFilter(extendedAllGroupsSearchFilter);
+
+ try {
+ sink.postUserGroupAuditInfo(ugsyncAuditInfo);
+ } catch (Throwable t) {
+ LOG.error("sink.postUserGroupAuditInfo failed with exception: " +
t.getMessage());
Review Comment:
Review log messages to replace string concat with parameterization.
##########
ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java:
##########
@@ -52,1240 +50,1161 @@
import javax.naming.ldap.StartTlsRequest;
import javax.naming.ldap.StartTlsResponse;
-import org.apache.commons.collections.MapUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.thirdparty.com.google.common.collect.HashBasedTable;
-import org.apache.hadoop.thirdparty.com.google.common.collect.Table;
-import org.apache.ranger.ugsyncutil.util.UgsyncCommonConstants;
-import org.apache.ranger.unixusersync.config.UserGroupSyncConfig;
-import org.apache.ranger.ugsyncutil.model.LdapSyncSourceInfo;
-import org.apache.ranger.ugsyncutil.model.UgsyncAuditInfo;
-import org.apache.ranger.usergroupsync.UserGroupSink;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.ranger.usergroupsync.UserGroupSource;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Properties;
+import java.util.Set;
+import java.util.UUID;
public class LdapUserGroupBuilder implements UserGroupSource {
-
- private static final Logger LOG =
LoggerFactory.getLogger(LdapUserGroupBuilder.class);
-
- private UserGroupSyncConfig config = UserGroupSyncConfig.getInstance();
-
- private static final String DATA_TYPE_BYTEARRAY = "byte[]";
- private static final String DATE_FORMAT = "yyyyMMddHHmmss";
- private static final int PAGE_SIZE = 500;
- private static final String MEMBER_OF_ATTR = "memberof=";
- private static final String GROUP_NAME_ATTRIBUTE = "cn=";
- private static long deltaSyncUserTime = 0; // Used for AD uSNChanged
- private static long deltaSyncGroupTime = 0; // Used for AD uSNChanged
- private String deltaSyncUserTimeStamp; // Used for OpenLdap
modifyTimestamp
- private String deltaSyncGroupTimeStamp; // Used for OpenLdap
modifyTimestamp
-
- private String ldapUrl;
- private String ldapBindDn;
- private String ldapBindPassword;
- private String ldapAuthenticationMechanism;
- private String ldapReferral;
- private String searchBase;
-
- private String[] userSearchBase;
- private String userNameAttribute;
- private String userCloudIdAttribute;
- private int userSearchScope;
- private String userObjectClass;
- private String userSearchFilter;
- private Set<String> groupNameSet;
- private String extendedUserSearchFilter;
- private SearchControls userSearchControls;
- private Set<String> userGroupNameAttributeSet;
- private Set<String> otherUserAttributes;
-
- private boolean pagedResultsEnabled = true;
- private int pagedResultsSize = PAGE_SIZE;
-
- private boolean groupSearchFirstEnabled = true;
- private boolean userSearchEnabled = true;
- private boolean groupSearchEnabled = true;
- private String[] groupSearchBase;
- private int groupSearchScope;
- private String groupObjectClass;
- private String groupSearchFilter;
- private String extendedGroupSearchFilter;
- private String extendedAllGroupsSearchFilter;
- private SearchControls groupSearchControls;
- private String groupMemberAttributeName;
- private String groupNameAttribute;
- private String groupCloudIdAttribute;
- private Set<String> otherGroupAttributes;
- private int groupHierarchyLevels;
- private int deleteCycles;
- private String currentSyncSource;
-
- private LdapContext ldapContext;
- StartTlsResponse tls;
-
- private Table<String, String, String> groupUserTable;
- UgsyncAuditInfo ugsyncAuditInfo;
- LdapSyncSourceInfo ldapSyncSourceInfo;
-
- private Map<String, Map<String, String>> sourceUsers; // key is user DN
and value is map of user attributes containing original name, DN, etc...
- private Map<String, Map<String, String>> sourceGroups; // key is group
DN and value is map of group attributes containing original name, DN, etc...
- private Map<String, Set<String>> sourceGroupUsers; // key is group DN
and value is set of user DNs (members)
-
- public static void main(String[] args) throws Throwable {
- LdapUserGroupBuilder ugBuilder = new LdapUserGroupBuilder();
- ugBuilder.init();
- }
-
- @Override
- public void init() throws Throwable{
- deltaSyncUserTime = 0;
- deltaSyncGroupTime = 0;
- deleteCycles = 1;
- DateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT);
- deltaSyncUserTimeStamp = dateFormat.format(new Date(0));
- deltaSyncGroupTimeStamp = dateFormat.format(new Date(0));
- setConfig();
- ugsyncAuditInfo = new UgsyncAuditInfo();
- ldapSyncSourceInfo = new LdapSyncSourceInfo();
- ldapSyncSourceInfo.setLdapUrl(ldapUrl);
-
ldapSyncSourceInfo.setIncrementalSycn(Boolean.toString(config.isDeltaSyncEnabled()));
-
ldapSyncSourceInfo.setUserSearchEnabled(Boolean.toString(userSearchEnabled));
-
ldapSyncSourceInfo.setGroupSearchEnabled(Boolean.toString(groupSearchEnabled));
-
ldapSyncSourceInfo.setGroupSearchFirstEnabled(Boolean.toString(groupSearchFirstEnabled));
-
ldapSyncSourceInfo.setGroupHierarchyLevel(Integer.toString(groupHierarchyLevels));
- ugsyncAuditInfo.setSyncSource(currentSyncSource);
- ugsyncAuditInfo.setLdapSyncSourceInfo(ldapSyncSourceInfo);
- }
-
- private void createLdapContext() throws Throwable {
- Properties env = new Properties();
- env.put(Context.INITIAL_CONTEXT_FACTORY,
- "com.sun.jndi.ldap.LdapCtxFactory");
- env.put(Context.PROVIDER_URL, ldapUrl);
- if (ldapUrl.startsWith("ldaps") &&
(config.getSSLTrustStorePath() != null &&
!config.getSSLTrustStorePath().trim().isEmpty())) {
- env.put("java.naming.ldap.factory.socket",
"org.apache.ranger.ldapusersync.process.CustomSSLSocketFactory");
- }
-
- if (StringUtils.isNotEmpty(userCloudIdAttribute)) {
- if
(config.getUserCloudIdAttributeDataType().equals(DATA_TYPE_BYTEARRAY)) {
- env.put("java.naming.ldap.attributes.binary",
userCloudIdAttribute);
- }
- }
-
- if (StringUtils.isNotEmpty(groupCloudIdAttribute)) {
- if
(config.getGroupCloudIdAttributeDataType().equals(DATA_TYPE_BYTEARRAY)) {
- env.put("java.naming.ldap.attributes.binary",
groupCloudIdAttribute);
- }
- }
-
- for (String otherUserAttribute : otherUserAttributes) {
- String attrType =
config.getOtherUserAttributeDataType(otherUserAttribute);
- if (attrType.equals(DATA_TYPE_BYTEARRAY)) {
- env.put("java.naming.ldap.attributes.binary",
otherUserAttribute);
- }
- }
-
- for (String otherGroupAttribute : otherGroupAttributes) {
- String attrType =
config.getOtherGroupAttributeDataType(otherGroupAttribute);
- if (attrType.equals(DATA_TYPE_BYTEARRAY)) {
- env.put("java.naming.ldap.attributes.binary",
otherGroupAttribute);
- }
- }
- ldapContext = new InitialLdapContext(env, null);
- if (!ldapUrl.startsWith("ldaps")) {
- if (config.isStartTlsEnabled()) {
- tls = (StartTlsResponse)
ldapContext.extendedOperation(new StartTlsRequest());
- if (config.getSSLTrustStorePath() != null &&
!config.getSSLTrustStorePath().trim().isEmpty()) {
-
tls.negotiate(CustomSSLSocketFactory.getDefault());
- } else {
- tls.negotiate();
- }
- LOG.info("Starting TLS session...");
- }
- }
-
- ldapContext.addToEnvironment(Context.SECURITY_PRINCIPAL,
ldapBindDn);
- ldapContext.addToEnvironment(Context.SECURITY_CREDENTIALS,
ldapBindPassword);
- ldapContext.addToEnvironment(Context.SECURITY_AUTHENTICATION,
ldapAuthenticationMechanism);
- ldapContext.addToEnvironment(Context.REFERRAL, ldapReferral);
- }
-
- private void setConfig() throws Throwable {
- LOG.info("LdapUserGroupBuilder initialization started");
-
- currentSyncSource = config.getCurrentSyncSource();
- groupSearchFirstEnabled = true;
- userSearchEnabled = config.isUserSearchEnabled();
- groupSearchEnabled = config.isGroupSearchEnabled();
- ldapUrl = config.getLdapUrl();
- ldapBindDn = config.getLdapBindDn();
- ldapBindPassword = config.getLdapBindPassword();
- //ldapBindPassword = "admin-password";
- ldapAuthenticationMechanism =
config.getLdapAuthenticationMechanism();
- ldapReferral = config.getContextReferral();
- searchBase = config.getSearchBase();
-
- userSearchBase = config.getUserSearchBase().split(";");
- userSearchScope = config.getUserSearchScope();
- userObjectClass = config.getUserObjectClass();
- userSearchFilter = config.getUserSearchFilter();
- userNameAttribute = config.getUserNameAttribute();
- userCloudIdAttribute = config.getUserCloudIdAttribute();
-
- Set<String> userSearchAttributes = new HashSet<String>();
- userSearchAttributes.add(userNameAttribute);
- userGroupNameAttributeSet =
config.getUserGroupNameAttributeSet();
- for (String useGroupNameAttribute : userGroupNameAttributeSet) {
- userSearchAttributes.add(useGroupNameAttribute);
- }
- userSearchAttributes.add(userCloudIdAttribute);
- otherUserAttributes = config.getOtherUserAttributes();
- for (String otherUserAttribute : otherUserAttributes) {
- userSearchAttributes.add(otherUserAttribute);
- }
- userSearchAttributes.add("uSNChanged");
- userSearchAttributes.add("modifytimestamp");
- userSearchControls = new SearchControls();
- userSearchControls.setSearchScope(userSearchScope);
-
userSearchControls.setReturningAttributes(userSearchAttributes.toArray(
- new String[userSearchAttributes.size()]));
-
- pagedResultsEnabled = config.isPagedResultsEnabled();
- pagedResultsSize = config.getPagedResultsSize();
-
- groupSearchBase = config.getGroupSearchBase().split(";");
- groupSearchScope = config.getGroupSearchScope();
- groupObjectClass = config.getGroupObjectClass();
- groupSearchFilter = config.getGroupSearchFilter();
- groupMemberAttributeName =
config.getUserGroupMemberAttributeName();
- groupNameAttribute = config.getGroupNameAttribute();
- groupCloudIdAttribute = config.getGroupCloudIdAttribute();
- groupHierarchyLevels = config.getGroupHierarchyLevels();
-
- extendedGroupSearchFilter = "(&" + extendedGroupSearchFilter +
"(|(" + groupMemberAttributeName + "={0})(" + groupMemberAttributeName +
"={1})))";
-
- groupSearchControls = new SearchControls();
- groupSearchControls.setSearchScope(groupSearchScope);
-
- Set<String> groupSearchAttributes = new HashSet<String>();
- groupSearchAttributes.add(groupNameAttribute);
- groupSearchAttributes.add(groupCloudIdAttribute);
- groupSearchAttributes.add(groupMemberAttributeName);
- groupSearchAttributes.add("uSNChanged");
- groupSearchAttributes.add("modifytimestamp");
- otherGroupAttributes = config.getOtherGroupAttributes();
- for (String otherGroupAttribute : otherGroupAttributes) {
- groupSearchAttributes.add(otherGroupAttribute);
- }
-
groupSearchControls.setReturningAttributes(groupSearchAttributes.toArray(
- new String[groupSearchAttributes.size()]));
-
- if (StringUtils.isEmpty(userSearchFilter)) {
- groupNameSet = config.getGroupNameSet();
- String computedSearchFilter = "";
- for (String groupName : groupNameSet) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("groupName = " + groupName);
- }
- if (!groupName.startsWith(MEMBER_OF_ATTR) &&
!groupName.startsWith(GROUP_NAME_ATTRIBUTE)){
- LOG.info("Ignoring unsupported format
for " + groupName );
- continue;
- }
- String searchFilter = groupName;
- if (groupName.startsWith(MEMBER_OF_ATTR)) {
- searchFilter =
groupName.substring(MEMBER_OF_ATTR.length());
- }
- searchFilter = getFirstRDN(searchFilter);
- computedSearchFilter +=
getDNForMemberOf(searchFilter);
- }
- if (StringUtils.isNotEmpty(computedSearchFilter)) {
- computedSearchFilter = "(|" +
computedSearchFilter + ")";
- }
- LOG.info("Final computedSearchFilter = " +
computedSearchFilter);
- userSearchFilter = computedSearchFilter;
- }
-
- LOG.info("LdapUserGroupBuilder initialization completed with --
"
- + "ldapUrl: " + ldapUrl
- + ", ldapBindDn: " + ldapBindDn
- + ", ldapBindPassword: ***** "
- + ", ldapAuthenticationMechanism: " +
ldapAuthenticationMechanism
- + ", searchBase: " + searchBase
- + ", userSearchBase: " +
Arrays.toString(userSearchBase)
- + ", userSearchScope: " + userSearchScope
- + ", userObjectClass: " + userObjectClass
- + ", userSearchFilter: " + userSearchFilter
- + ", extendedUserSearchFilter: " +
extendedUserSearchFilter
- + ", userNameAttribute: " + userNameAttribute
- + ", userSearchAttributes: " +
userSearchAttributes
- + ", userGroupNameAttributeSet: " +
userGroupNameAttributeSet
- + ", otherUserAttributes: " +
otherUserAttributes
- + ", pagedResultsEnabled: " +
pagedResultsEnabled
- + ", pagedResultsSize: " + pagedResultsSize
- + ", groupSearchEnabled: " + groupSearchEnabled
- + ", groupSearchBase: " +
Arrays.toString(groupSearchBase)
- + ", groupSearchScope: " + groupSearchScope
- + ", groupObjectClass: " + groupObjectClass
- + ", groupSearchFilter: " + groupSearchFilter
- + ", extendedGroupSearchFilter: " +
extendedGroupSearchFilter
- + ", extendedAllGroupsSearchFilter: " +
extendedAllGroupsSearchFilter
- + ", groupMemberAttributeName: " +
groupMemberAttributeName
- + ", groupNameAttribute: " + groupNameAttribute
- + ", groupSearchAttributes: " +
groupSearchAttributes
- + ", groupSearchFirstEnabled: " +
groupSearchFirstEnabled
- + ", userSearchEnabled: " + userSearchEnabled
- + ", ldapReferral: " + ldapReferral
- );
- }
-
- private void closeLdapContext() throws Throwable {
- if (tls != null) {
- tls.close();
- }
- if (ldapContext != null) {
- ldapContext.close();
- }
- }
-
- @Override
- public boolean isChanged() {
- // we do not want to get the full ldap dit and check whether
anything has changed
- return true;
- }
-
- @Override
- public void updateSink(UserGroupSink sink) throws Throwable {
- LOG.info("LdapUserGroupBuilder updateSink started");
+ private static final Logger LOG =
LoggerFactory.getLogger(LdapUserGroupBuilder.class);
+ private static final String DATA_TYPE_BYTEARRAY = "byte[]";
+ private static final String DATE_FORMAT = "yyyyMMddHHmmss";
+ private static final String MEMBER_OF_ATTR = "memberof=";
+ private static final String GROUP_NAME_ATTRIBUTE = "cn=";
+ private static final int PAGE_SIZE = 500;
+ /* for AD uSNChanged */
+ private static long deltaSyncUserTime;
+ private static long deltaSyncGroupTime;
+ /* ***************** */
+ private boolean pagedResultsEnabled = true;
+ private boolean groupSearchFirstEnabled = true;
+ private boolean userSearchEnabled = true;
+ private boolean groupSearchEnabled = true;
+ private int pagedResultsSize = PAGE_SIZE;
+ private int groupHierarchyLevels;
+ private int deleteCycles;
+ private int userSearchScope;
+ private int groupSearchScope;
+
+ private UserGroupSyncConfig config = UserGroupSyncConfig.getInstance();
+ /* for OpenLdap modifyTimestamp */
+ private String deltaSyncUserTimeStamp;
+ private String deltaSyncGroupTimeStamp;
+ /* ******************************** */
+ private String ldapUrl;
+ private String ldapBindDn;
+ private String ldapBindPassword;
+ private String ldapAuthenticationMechanism;
+ private String ldapReferral;
+ private String searchBase;
+ private String userNameAttribute;
+ private String userCloudIdAttribute;
+ private String userObjectClass;
+ private String userSearchFilter;
+ private String extendedUserSearchFilter;
+ private String groupObjectClass;
+ private String groupSearchFilter;
+ private String extendedGroupSearchFilter;
+ private String extendedAllGroupsSearchFilter;
+ private String groupMemberAttributeName;
+ private String groupNameAttribute;
+ private String groupCloudIdAttribute;
+ private String currentSyncSource;
+ private String[] userSearchBase;
+ private String[] groupSearchBase;
+ private Set<String> groupNameSet;
+ private Set<String> userGroupNameAttributeSet;
+ private Set<String> otherUserAttributes;
+ private Set<String> otherGroupAttributes;
+ private LdapContext ldapContext;
+ private SearchControls userSearchControls;
+ private SearchControls groupSearchControls;
+ private Table<String, String, String> groupUserTable;
+ /* { key = user DN, value = map of user attributes {original name, DN,
etc.}} */
+ private Map<String, Map<String, String>> sourceUsers;
+ /* { key = group DN, value = map of group attributes {original name, DN,
etc.}} */
+ private Map<String, Map<String, String>> sourceGroups;
+ /* { key = group DN, value = set of user DNs (members) } */
+ private Map<String, Set<String>> sourceGroupUsers;
+
+ StartTlsResponse tls;
+ UgsyncAuditInfo ugsyncAuditInfo;
+ LdapSyncSourceInfo ldapSyncSourceInfo;
+
+ public static void main(String[] args) throws Throwable {
+ LdapUserGroupBuilder ugBuilder = new LdapUserGroupBuilder();
+ ugBuilder.init();
+ }
+
+ @Override
+ public void init() throws Throwable {
+ deltaSyncUserTime = 0;
+ deltaSyncGroupTime = 0;
+ deleteCycles = 1;
+ DateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT);
+ deltaSyncUserTimeStamp = dateFormat.format(new Date(0));
+ deltaSyncGroupTimeStamp = dateFormat.format(new Date(0));
+ setConfig();
+ ugsyncAuditInfo = new UgsyncAuditInfo();
+ ldapSyncSourceInfo = new LdapSyncSourceInfo();
+ ldapSyncSourceInfo.setLdapUrl(ldapUrl);
+
ldapSyncSourceInfo.setIncrementalSycn(Boolean.toString(config.isDeltaSyncEnabled()));
+
ldapSyncSourceInfo.setUserSearchEnabled(Boolean.toString(userSearchEnabled));
+
ldapSyncSourceInfo.setGroupSearchEnabled(Boolean.toString(groupSearchEnabled));
+
ldapSyncSourceInfo.setGroupSearchFirstEnabled(Boolean.toString(groupSearchFirstEnabled));
+
ldapSyncSourceInfo.setGroupHierarchyLevel(Integer.toString(groupHierarchyLevels));
+ ugsyncAuditInfo.setSyncSource(currentSyncSource);
+ ugsyncAuditInfo.setLdapSyncSourceInfo(ldapSyncSourceInfo);
+ }
+
+ @Override
+ public boolean isChanged() {
+ // we do not want to get the full ldap dit and check whether anything
has changed
+ return true;
+ }
+
+ @Override
+ public void updateSink(UserGroupSink sink) throws Throwable {
+ LOG.info("LdapUserGroupBuilder updateSink started");
boolean computeDeletes = false;
- groupUserTable = HashBasedTable.create();
- sourceGroups = new HashMap<>();
- sourceUsers = new HashMap<>();
- sourceGroupUsers = new HashMap<>();
- long highestdeltaSyncUserTime = 0;
- long highestdeltaSyncGroupTime = 0;
-
- if (config.isUserSyncDeletesEnabled() && deleteCycles >=
config.getUserSyncDeletesFrequency()) {
- deleteCycles = 1;
- computeDeletes = true;
- if (LOG.isDebugEnabled()) {
- LOG.debug("Compute deleted users/groups is
enabled for this sync cycle");
- }
- }
- if (config.isUserSyncDeletesEnabled()) {
- deleteCycles++;
- }
- if (groupSearchEnabled) {
- highestdeltaSyncGroupTime = getGroups(computeDeletes);
- }
- if (userSearchEnabled) {
- LOG.info("Performing user search to retrieve users from
AD/LDAP");
- highestdeltaSyncUserTime = getUsers(computeDeletes);
- }
-
- if (groupHierarchyLevels > 0) {
- LOG.info("Going through group hierarchy for nested
group evaluation");
+ groupUserTable = HashBasedTable.create();
+ sourceGroups = new HashMap<>();
+ sourceUsers = new HashMap<>();
+ sourceGroupUsers = new HashMap<>();
+ long highestdeltaSyncUserTime = 0;
+ long highestdeltaSyncGroupTime = 0;
+
+ if (config.isUserSyncDeletesEnabled() && deleteCycles >=
config.getUserSyncDeletesFrequency()) {
+ deleteCycles = 1;
+ computeDeletes = true;
+ LOG.debug("Compute deleted users/groups is enabled for this sync
cycle");
+ }
+ if (config.isUserSyncDeletesEnabled()) {
+ deleteCycles++;
+ }
+ if (groupSearchEnabled) {
+ highestdeltaSyncGroupTime = getGroups(computeDeletes);
+ }
+ if (userSearchEnabled) {
+ LOG.info("Performing user search to retrieve users from AD/LDAP");
+ highestdeltaSyncUserTime = getUsers(computeDeletes);
+ }
+
+ if (groupHierarchyLevels > 0) {
+ LOG.info("Going through group hierarchy for nested group
evaluation");
Set<String> groupFullNames = sourceGroups.keySet();
- for(String group : groupFullNames) {
- Set<String> nextLevelGroups =
groupUserTable.column(group).keySet();
- goUpGroupHierarchy(nextLevelGroups,
groupHierarchyLevels-1, group);
- }
- LOG.info("Completed group hierarchy computation");
- }
-
- Iterator<String> groupUserTableIterator =
groupUserTable.rowKeySet().iterator();
- while (groupUserTableIterator.hasNext()) {
- String groupName = groupUserTableIterator.next();
- Map<String,String> groupUsersMap =
groupUserTable.row(groupName);
- Set<String> userSet = new HashSet<String>();
- for(Map.Entry<String, String> entry :
groupUsersMap.entrySet()){
- if (sourceUsers.containsKey(entry.getValue())) {
- userSet.add(entry.getValue());
- }
- }
- sourceGroupUsers.put(groupName, userSet);
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Users = " + sourceUsers.keySet());
- LOG.debug("Groups = " + sourceGroups.keySet());
- LOG.debug("GroupUsers = " + sourceGroupUsers.keySet());
- }
-
- try {
- sink.addOrUpdateUsersGroups(sourceGroups, sourceUsers,
sourceGroupUsers, computeDeletes);
- DateFormat dateFormat = new
SimpleDateFormat(DATE_FORMAT);
- LOG.info("deltaSyncUserTime = " + deltaSyncUserTime + "
and highestdeltaSyncUserTime = " + highestdeltaSyncUserTime);
- if (deltaSyncUserTime < highestdeltaSyncUserTime) {
- // Incrementing highestdeltaSyncUserTime (for
AD) in order to avoid search record repetition for next sync cycle.
- deltaSyncUserTime = highestdeltaSyncUserTime +
1;
- // Incrementing the highest timestamp value
(for Openldap) with 1sec in order to avoid search record repetition for next
sync cycle.
- deltaSyncUserTimeStamp = dateFormat.format(new
Date(highestdeltaSyncUserTime + 60l));
- }
-
- LOG.info("deltaSyncGroupTime = " + deltaSyncGroupTime +
" and highestdeltaSyncGroupTime = " + highestdeltaSyncGroupTime);
- // Update deltaSyncUserTime/deltaSyncUserTimeStamp here
so that in case of failures, we get updates in next cycle
- if (deltaSyncGroupTime < highestdeltaSyncGroupTime) {
- // Incrementing highestdeltaSyncGroupTime (for
AD) in order to avoid search record repetition for next sync cycle.
- deltaSyncGroupTime =
highestdeltaSyncGroupTime+1;
- // Incrementing the highest timestamp value
(for OpenLdap) with 1min in order to avoid search record repetition for next
sync cycle.
- deltaSyncGroupTimeStamp = dateFormat.format(new
Date(highestdeltaSyncGroupTime + 60l));
- }
- } catch (Throwable t) {
- LOG.error("Failed to update ranger admin. Will retry in
next sync cycle!!", t);
- }
-
-
ldapSyncSourceInfo.setUserSearchFilter(extendedUserSearchFilter);
-
ldapSyncSourceInfo.setGroupSearchFilter(extendedAllGroupsSearchFilter);
-
- try {
- sink.postUserGroupAuditInfo(ugsyncAuditInfo);
- } catch (Throwable t) {
- LOG.error("sink.postUserGroupAuditInfo failed with
exception: " + t.getMessage());
- }
- }
-
- private long getUsers(boolean computeDeletes) throws Throwable {
- NamingEnumeration<SearchResult> userSearchResultEnum = null;
- NamingEnumeration<SearchResult> groupSearchResultEnum = null;
- long highestdeltaSyncUserTime;
- try {
- createLdapContext();
- int total;
- // Activate paged results
- if (pagedResultsEnabled) {
- ldapContext.setRequestControls(new Control[]{
- new
PagedResultsControl(pagedResultsSize, Control.NONCRITICAL) });
- }
- DateFormat dateFormat = new
SimpleDateFormat(DATE_FORMAT);
- if (groupUserTable.rowKeySet().size() != 0 ||
!config.isDeltaSyncEnabled() || (computeDeletes)) {
- // Fix RANGER-1957: Perform full sync when
there are updates to the groups or when incremental sync is not enabled
- deltaSyncUserTime = 0;
- deltaSyncUserTimeStamp = dateFormat.format(new
Date(0));
- }
-
- extendedUserSearchFilter = "(objectclass=" +
userObjectClass + ")(|(uSNChanged>=" + deltaSyncUserTime +
")(modifyTimestamp>=" + deltaSyncUserTimeStamp + "Z))";
-
- if (userSearchFilter != null &&
!userSearchFilter.trim().isEmpty()) {
- String customFilter = userSearchFilter.trim();
- if (!customFilter.startsWith("(")) {
- customFilter = "(" + customFilter + ")";
- }
-
- extendedUserSearchFilter = "(&" +
extendedUserSearchFilter + customFilter + ")";
- } else {
- extendedUserSearchFilter = "(&" +
extendedUserSearchFilter + ")";
- }
- LOG.info("extendedUserSearchFilter = " +
extendedUserSearchFilter);
-
- highestdeltaSyncUserTime = deltaSyncUserTime;
-
- // When multiple OUs are configured, go through each OU
as the user search base to search for users.
- for (int ou=0; ou<userSearchBase.length; ou++) {
- byte[] cookie = null;
- int counter = 0;
- try {
- int paged = 0;
- do {
- userSearchResultEnum = ldapContext
-
.search(userSearchBase[ou], extendedUserSearchFilter,
-
userSearchControls);
-
- while (userSearchResultEnum.hasMore()) {
- // searchResults contains all
the user entries
- final SearchResult userEntry =
userSearchResultEnum.next();
-
- if (userEntry == null) {
- LOG.info("userEntry
null, skipping sync for the entry");
- continue;
- }
-
- Attributes attributes =
userEntry.getAttributes();
- if (attributes == null) {
- LOG.info("attributes
missing for entry " + userEntry.getNameInNamespace() +
- ",
skipping sync");
- continue;
- }
-
- Attribute userNameAttr =
attributes.get(userNameAttribute);
- if (userNameAttr == null) {
-
LOG.info(userNameAttribute + " missing for entry " +
userEntry.getNameInNamespace() +
- ",
skipping sync");
- continue;
- }
-
- String userFullName =
(userEntry.getNameInNamespace());
- String userName = (String)
userNameAttr.get();
-
- if (userName == null ||
userName.trim().isEmpty()) {
-
LOG.info(userNameAttribute + " empty for entry " +
userEntry.getNameInNamespace() +
- ",
skipping sync");
- continue;
- }
-
- Attribute timeStampAttr =
attributes.get("uSNChanged");
- if (timeStampAttr != null) {
- String uSNChangedVal =
(String) timeStampAttr.get();
- long
currentDeltaSyncTime = Long.parseLong(uSNChangedVal);
- LOG.info("uSNChangedVal
= " + uSNChangedVal + "and currentDeltaSyncTime = " + currentDeltaSyncTime);
- if
(currentDeltaSyncTime > highestdeltaSyncUserTime) {
-
highestdeltaSyncUserTime = currentDeltaSyncTime;
- }
- } else {
- timeStampAttr =
attributes.get("modifytimestamp");
- if (timeStampAttr !=
null) {
- String
timeStampVal = (String) timeStampAttr.get();
- Date parseDate
= dateFormat.parse(timeStampVal);
- long
currentDeltaSyncTime = parseDate.getTime();
-
LOG.info("timeStampVal = " + timeStampVal + "and currentDeltaSyncTime = " +
currentDeltaSyncTime);
- if
(currentDeltaSyncTime > highestdeltaSyncUserTime) {
-
highestdeltaSyncUserTime = currentDeltaSyncTime;
-
deltaSyncUserTimeStamp = timeStampVal;
- }
- }
- }
-
- // Get all the groups from the
group name attribute of the user only when group search is not enabled.
- if (!groupSearchEnabled) {
- for (String
useGroupNameAttribute : userGroupNameAttributeSet) {
- Attribute
userGroupfAttribute = userEntry.getAttributes().get(useGroupNameAttribute);
- if
(userGroupfAttribute != null) {
-
NamingEnumeration<?> groupEnum = userGroupfAttribute.getAll();
- while
(groupEnum.hasMore()) {
-
String groupDN = (String) groupEnum.next();
-
if (LOG.isDebugEnabled()) {
-
LOG.debug("Adding " + groupDN + " to " + userName);
-
}
-
Map<String, String> groupAttrMap = new HashMap<>();
-
String groupName = getShortName(groupDN);
-
groupAttrMap.put(UgsyncCommonConstants.ORIGINAL_NAME, groupName);
-
groupAttrMap.put(UgsyncCommonConstants.FULL_NAME, groupDN);
-
groupAttrMap.put(UgsyncCommonConstants.SYNC_SOURCE, currentSyncSource);
-
groupAttrMap.put(UgsyncCommonConstants.LDAP_URL, config.getLdapUrl());
-
sourceGroups.put(groupDN, groupAttrMap);
-
if (LOG.isDebugEnabled()) {
-
LOG.debug("As groupsearch is disabled, adding group " + groupName + "
from user memberof attribute for user " + userName);
-
}
-
groupUserTable.put(groupDN, userFullName, userFullName);
- }
- }
- }
- }
-
- Map<String, String> userAttrMap
= new HashMap<>();
-
userAttrMap.put(UgsyncCommonConstants.ORIGINAL_NAME, userName);
-
userAttrMap.put(UgsyncCommonConstants.FULL_NAME, userFullName);
-
userAttrMap.put(UgsyncCommonConstants.SYNC_SOURCE, currentSyncSource);
-
userAttrMap.put(UgsyncCommonConstants.LDAP_URL, config.getLdapUrl());
- Attribute userCloudIdAttr =
attributes.get(userCloudIdAttribute);
- if (userCloudIdAttr != null) {
-
addToAttrMap(userAttrMap, "cloud_id", userCloudIdAttr,
config.getUserCloudIdAttributeDataType());
- }
- for (String otherUserAttribute
: otherUserAttributes) {
- if
(attributes.get(otherUserAttribute) != null) {
- String attrType
= config.getOtherUserAttributeDataType(otherUserAttribute);
-
addToAttrMap(userAttrMap, otherUserAttribute,
attributes.get(otherUserAttribute), attrType);
- }
- }
-
- sourceUsers.put(userFullName,
userAttrMap);
- if
((groupUserTable.containsColumn(userFullName) ||
groupUserTable.containsColumn(userName))) {
- //Update the username
in the groupUserTable with the one from username attribute.
- Map<String, String>
userMap = groupUserTable.column(userFullName);
- if
(MapUtils.isEmpty(userMap)) {
- userMap =
groupUserTable.column(userName);
- }
- for (Map.Entry<String,
String> entry : userMap.entrySet()) {
- if
(LOG.isDebugEnabled()) {
-
LOG.debug("Updating groupUserTable " + entry.getValue() + " with: " + userName
+ " for " + entry.getKey());
- }
-
groupUserTable.put(entry.getKey(), userFullName, userFullName);
- }
- }
- counter++;
-
- if (counter <= 2000) {
- LOG.info("Updating user count: " + counter + ",
userName: " + userName);
- if ( counter == 2000 ) {
- LOG.info("===> 2000 user records have been
synchronized so far. From now on, only a summary progress log will be written
for every 100 users. To continue to see detailed log for every user, please
enable Trace level logging. <===");
+ for (String group : groupFullNames) {
+ Set<String> nextLevelGroups =
groupUserTable.column(group).keySet();
+ goUpGroupHierarchy(nextLevelGroups, groupHierarchyLevels - 1,
group);
+ }
+ LOG.info("Completed group hierarchy computation");
+ }
+
+ Iterator<String> groupUserTableIterator =
groupUserTable.rowKeySet().iterator();
+ while (groupUserTableIterator.hasNext()) {
+ String groupName = groupUserTableIterator.next();
+ Map<String, String> groupUsersMap = groupUserTable.row(groupName);
+ Set<String> userSet = new HashSet<String>();
+ for (Map.Entry<String, String> entry : groupUsersMap.entrySet()) {
+ if (sourceUsers.containsKey(entry.getValue())) {
+ userSet.add(entry.getValue());
+ }
+ }
+ sourceGroupUsers.put(groupName, userSet);
+ }
+
+ LOG.debug("Users = {}", sourceUsers.keySet());
+ LOG.debug("Groups = {}", sourceGroups.keySet());
+ LOG.debug("GroupUsers = {}", sourceGroupUsers.keySet());
+
+ try {
+ sink.addOrUpdateUsersGroups(sourceGroups, sourceUsers,
sourceGroupUsers, computeDeletes);
+ DateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT);
+ LOG.info("deltaSyncUserTime = {} and highestdeltaSyncUserTime =
{}", deltaSyncUserTime, highestdeltaSyncUserTime);
+ if (deltaSyncUserTime < highestdeltaSyncUserTime) {
+ // Incrementing highestdeltaSyncUserTime (for AD) in order to
avoid search record repetition for next sync cycle.
+ deltaSyncUserTime = highestdeltaSyncUserTime + 1;
+ // Incrementing the highest timestamp value (for Openldap)
with 1sec in order to avoid search record repetition for next sync cycle.
+ deltaSyncUserTimeStamp = dateFormat.format(new
Date(highestdeltaSyncUserTime + 60L));
+ }
+
+ LOG.info("deltaSyncGroupTime = {} and highestdeltaSyncGroupTime =
{} ", deltaSyncGroupTime, highestdeltaSyncGroupTime);
+ // Update deltaSyncUserTime/deltaSyncUserTimeStamp here so that in
case of failures, we get updates in next cycle
+ if (deltaSyncGroupTime < highestdeltaSyncGroupTime) {
+ // Incrementing highestdeltaSyncGroupTime (for AD) in order to
avoid search record repetition for next sync cycle.
+ deltaSyncGroupTime = highestdeltaSyncGroupTime + 1;
+ // Incrementing the highest timestamp value (for OpenLdap)
with 1min in order to avoid search record repetition for next sync cycle.
+ deltaSyncGroupTimeStamp = dateFormat.format(new
Date(highestdeltaSyncGroupTime + 60L));
+ }
+ } catch (Throwable t) {
+ LOG.error("Failed to update ranger admin. Will retry in next sync
cycle!!", t);
+ }
+
+ ldapSyncSourceInfo.setUserSearchFilter(extendedUserSearchFilter);
+ ldapSyncSourceInfo.setGroupSearchFilter(extendedAllGroupsSearchFilter);
+
+ try {
+ sink.postUserGroupAuditInfo(ugsyncAuditInfo);
+ } catch (Throwable t) {
+ LOG.error("sink.postUserGroupAuditInfo failed with exception: " +
t.getMessage());
+ }
+ }
+
+ private void createLdapContext() throws Throwable {
+ Properties env = new Properties();
+ env.put(Context.INITIAL_CONTEXT_FACTORY,
"com.sun.jndi.ldap.LdapCtxFactory");
+ env.put(Context.PROVIDER_URL, ldapUrl);
+ if (ldapUrl.startsWith("ldaps") && (config.getSSLTrustStorePath() !=
null && !config.getSSLTrustStorePath().trim().isEmpty())) {
+ env.put("java.naming.ldap.factory.socket",
"org.apache.ranger.ldapusersync.process.CustomSSLSocketFactory");
+ }
+
+ if (StringUtils.isNotEmpty(userCloudIdAttribute)) {
+ if
(config.getUserCloudIdAttributeDataType().equals(DATA_TYPE_BYTEARRAY)) {
+ env.put("java.naming.ldap.attributes.binary",
userCloudIdAttribute);
+ }
+ }
+
+ if (StringUtils.isNotEmpty(groupCloudIdAttribute)) {
+ if
(config.getGroupCloudIdAttributeDataType().equals(DATA_TYPE_BYTEARRAY)) {
+ env.put("java.naming.ldap.attributes.binary",
groupCloudIdAttribute);
+ }
+ }
+
+ for (String otherUserAttribute : otherUserAttributes) {
+ String attrType =
config.getOtherUserAttributeDataType(otherUserAttribute);
+ if (attrType.equals(DATA_TYPE_BYTEARRAY)) {
+ env.put("java.naming.ldap.attributes.binary",
otherUserAttribute);
+ }
+ }
+
+ for (String otherGroupAttribute : otherGroupAttributes) {
+ String attrType =
config.getOtherGroupAttributeDataType(otherGroupAttribute);
+ if (attrType.equals(DATA_TYPE_BYTEARRAY)) {
+ env.put("java.naming.ldap.attributes.binary",
otherGroupAttribute);
+ }
+ }
+ ldapContext = new InitialLdapContext(env, null);
+ if (!ldapUrl.startsWith("ldaps")) {
+ if (config.isStartTlsEnabled()) {
+ tls = (StartTlsResponse) ldapContext.extendedOperation(new
StartTlsRequest());
+ if (config.getSSLTrustStorePath() != null &&
!config.getSSLTrustStorePath().trim().isEmpty()) {
+ tls.negotiate(CustomSSLSocketFactory.getDefault());
+ } else {
+ tls.negotiate();
+ }
+ LOG.info("Starting TLS session...");
+ }
+ }
+
+ ldapContext.addToEnvironment(Context.SECURITY_PRINCIPAL, ldapBindDn);
+ ldapContext.addToEnvironment(Context.SECURITY_CREDENTIALS,
ldapBindPassword);
+ ldapContext.addToEnvironment(Context.SECURITY_AUTHENTICATION,
ldapAuthenticationMechanism);
+ ldapContext.addToEnvironment(Context.REFERRAL, ldapReferral);
+ }
+
+ private void setConfig() throws Throwable {
+ LOG.info("LdapUserGroupBuilder initialization started");
+ groupSearchFirstEnabled = true;
+ currentSyncSource = config.getCurrentSyncSource();
+ userSearchEnabled = config.isUserSearchEnabled();
+ groupSearchEnabled = config.isGroupSearchEnabled();
+ ldapUrl = config.getLdapUrl();
+ ldapBindDn = config.getLdapBindDn();
+ ldapBindPassword = config.getLdapBindPassword();
+ ldapAuthenticationMechanism = config.getLdapAuthenticationMechanism();
+ ldapReferral = config.getContextReferral();
+ searchBase = config.getSearchBase();
+ userSearchBase = config.getUserSearchBase().split(";");
+ userSearchScope = config.getUserSearchScope();
+ userObjectClass = config.getUserObjectClass();
+ userSearchFilter = config.getUserSearchFilter();
+ userNameAttribute = config.getUserNameAttribute();
+ userCloudIdAttribute = config.getUserCloudIdAttribute();
+
+ Set<String> userSearchAttributes = new HashSet<String>();
+ userSearchAttributes.add(userNameAttribute);
+ userGroupNameAttributeSet = config.getUserGroupNameAttributeSet();
+ for (String useGroupNameAttribute : userGroupNameAttributeSet) {
+ userSearchAttributes.add(useGroupNameAttribute);
+ }
+ userSearchAttributes.add(userCloudIdAttribute);
+ otherUserAttributes = config.getOtherUserAttributes();
+ for (String otherUserAttribute : otherUserAttributes) {
+ userSearchAttributes.add(otherUserAttribute);
+ }
+ userSearchAttributes.add("uSNChanged");
+ userSearchAttributes.add("modifytimestamp");
+ userSearchControls = new SearchControls();
+ userSearchControls.setSearchScope(userSearchScope);
+
userSearchControls.setReturningAttributes(userSearchAttributes.toArray(new
String[userSearchAttributes.size()]));
+
+ pagedResultsEnabled = config.isPagedResultsEnabled();
+ pagedResultsSize = config.getPagedResultsSize();
+
+ groupSearchBase = config.getGroupSearchBase().split(";");
+ groupSearchScope = config.getGroupSearchScope();
+ groupObjectClass = config.getGroupObjectClass();
+ groupSearchFilter = config.getGroupSearchFilter();
+ groupMemberAttributeName = config.getUserGroupMemberAttributeName();
+ groupNameAttribute = config.getGroupNameAttribute();
+ groupCloudIdAttribute = config.getGroupCloudIdAttribute();
+ groupHierarchyLevels = config.getGroupHierarchyLevels();
+
+ extendedGroupSearchFilter = "(&" + extendedGroupSearchFilter + "(|(" +
groupMemberAttributeName + "={0})(" + groupMemberAttributeName + "={1})))";
+
+ groupSearchControls = new SearchControls();
+ groupSearchControls.setSearchScope(groupSearchScope);
+
+ Set<String> groupSearchAttributes = new HashSet<String>();
+ groupSearchAttributes.add(groupNameAttribute);
+ groupSearchAttributes.add(groupCloudIdAttribute);
+ groupSearchAttributes.add(groupMemberAttributeName);
+ groupSearchAttributes.add("uSNChanged");
+ groupSearchAttributes.add("modifytimestamp");
+ otherGroupAttributes = config.getOtherGroupAttributes();
+ for (String otherGroupAttribute : otherGroupAttributes) {
+ groupSearchAttributes.add(otherGroupAttribute);
+ }
+
groupSearchControls.setReturningAttributes(groupSearchAttributes.toArray(new
String[groupSearchAttributes.size()]));
+
+ if (StringUtils.isEmpty(userSearchFilter)) {
+ groupNameSet = config.getGroupNameSet();
+ String computedSearchFilter = "";
+ for (String groupName : groupNameSet) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("groupName = {}", groupName);
+ }
+ if (!groupName.startsWith(MEMBER_OF_ATTR) &&
!groupName.startsWith(GROUP_NAME_ATTRIBUTE)) {
+ LOG.info("Ignoring unsupported format for {}", groupName);
+ continue;
+ }
+ String searchFilter = groupName;
+ if (groupName.startsWith(MEMBER_OF_ATTR)) {
+ searchFilter =
groupName.substring(MEMBER_OF_ATTR.length());
+ }
+ searchFilter = getFirstRDN(searchFilter);
+ computedSearchFilter += getDNForMemberOf(searchFilter);
+ }
+ if (StringUtils.isNotEmpty(computedSearchFilter)) {
+ computedSearchFilter = "(|" + computedSearchFilter + ")";
+ }
+ LOG.info("Final computedSearchFilter = {}", computedSearchFilter);
+ userSearchFilter = computedSearchFilter;
+ }
+
+ LOG.info("LdapUserGroupBuilder initialization completed with -- "
+ + "ldapUrl: " + ldapUrl
+ + ", ldapBindDn: " + ldapBindDn
+ + ", ldapBindPassword: ***** "
+ + ", ldapAuthenticationMechanism: " +
ldapAuthenticationMechanism
+ + ", searchBase: " + searchBase
+ + ", userSearchBase: " + Arrays.toString(userSearchBase)
+ + ", userSearchScope: " + userSearchScope
+ + ", userObjectClass: " + userObjectClass
+ + ", userSearchFilter: " + userSearchFilter
+ + ", extendedUserSearchFilter: " + extendedUserSearchFilter
+ + ", userNameAttribute: " + userNameAttribute
+ + ", userSearchAttributes: " + userSearchAttributes
+ + ", userGroupNameAttributeSet: " + userGroupNameAttributeSet
+ + ", otherUserAttributes: " + otherUserAttributes
+ + ", pagedResultsEnabled: " + pagedResultsEnabled
+ + ", pagedResultsSize: " + pagedResultsSize
+ + ", groupSearchEnabled: " + groupSearchEnabled
+ + ", groupSearchBase: " + Arrays.toString(groupSearchBase)
+ + ", groupSearchScope: " + groupSearchScope
+ + ", groupObjectClass: " + groupObjectClass
+ + ", groupSearchFilter: " + groupSearchFilter
+ + ", extendedGroupSearchFilter: " + extendedGroupSearchFilter
+ + ", extendedAllGroupsSearchFilter: " +
extendedAllGroupsSearchFilter
+ + ", groupMemberAttributeName: " + groupMemberAttributeName
+ + ", groupNameAttribute: " + groupNameAttribute
+ + ", groupSearchAttributes: " + groupSearchAttributes
+ + ", groupSearchFirstEnabled: " + groupSearchFirstEnabled
+ + ", userSearchEnabled: " + userSearchEnabled
+ + ", ldapReferral: " + ldapReferral);
+ }
+
+ private void closeLdapContext() throws Throwable {
+ if (tls != null) {
+ tls.close();
+ }
+ if (ldapContext != null) {
+ ldapContext.close();
+ }
+ }
+
+ private long getUsers(boolean computeDeletes) throws Throwable {
+ NamingEnumeration<SearchResult> userSearchResultEnum = null;
+ NamingEnumeration<SearchResult> groupSearchResultEnum = null;
+ long highestdeltaSyncUserTime;
+ try {
+ createLdapContext();
+ int total;
+ // Activate paged results
+ if (pagedResultsEnabled) {
+ ldapContext.setRequestControls(new Control[] {new
PagedResultsControl(pagedResultsSize, Control.NONCRITICAL)});
+ }
+ DateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT);
+ if (groupUserTable.rowKeySet().size() != 0 ||
!config.isDeltaSyncEnabled() || (computeDeletes)) {
+ // Fix RANGER-1957: Perform full sync when there are updates
to the groups or when incremental sync is not enabled
+ deltaSyncUserTime = 0;
+ deltaSyncUserTimeStamp = dateFormat.format(new Date(0));
+ }
+
+ extendedUserSearchFilter = "(objectclass=" + userObjectClass +
")(|(uSNChanged>=" + deltaSyncUserTime + ")(modifyTimestamp>=" +
deltaSyncUserTimeStamp + "Z))";
+
+ if (userSearchFilter != null &&
!userSearchFilter.trim().isEmpty()) {
+ String customFilter = userSearchFilter.trim();
+ if (!customFilter.startsWith("(")) {
+ customFilter = "(" + customFilter + ")";
+ }
+
+ extendedUserSearchFilter = "(&" + extendedUserSearchFilter +
customFilter + ")";
+ } else {
+ extendedUserSearchFilter = "(&" + extendedUserSearchFilter +
")";
+ }
+ LOG.info("extendedUserSearchFilter = {}",
extendedUserSearchFilter);
+
+ highestdeltaSyncUserTime = deltaSyncUserTime;
+
+ // When multiple OUs are configured, go through each OU as the
user search base to search for users.
+ for (String s : userSearchBase) {
+ byte[] cookie = null;
+ int counter = 0;
+ try {
+ int paged = 0;
+ do {
+ userSearchResultEnum = ldapContext.search(s,
extendedUserSearchFilter, userSearchControls);
+
+ while (userSearchResultEnum.hasMore()) {
+ // searchResults contains all the user entries
+ final SearchResult userEntry =
userSearchResultEnum.next();
+
+ if (userEntry == null) {
+ LOG.info("userEntry null, skipping sync for
the entry");
+ continue;
}
- } else {
- if (LOG.isTraceEnabled()) {
- LOG.trace("Updating user count: " + counter
- + ", userName: " + userName);
- } else {
- if ( counter % 100 == 0) {
- LOG.info("Synced " + counter + " users
till now");
+
+ Attributes attributes = userEntry.getAttributes();
+ if (attributes == null) {
+ LOG.info("attributes missing for entry {},
skipping sync", userEntry.getNameInNamespace());
+ continue;
+ }
+
+ Attribute userNameAttr =
attributes.get(userNameAttribute);
+ if (userNameAttr == null) {
+ LOG.info("{} missing for entry {}, skipping
sync", userNameAttribute, userEntry.getNameInNamespace());
+ continue;
+ }
+
+ String userFullName =
userEntry.getNameInNamespace();
+ String userName = (String) userNameAttr.get();
+
+ if (userName == null || userName.trim().isEmpty())
{
+ LOG.info("{} empty for entry {}, skipping
sync", userNameAttribute, userEntry.getNameInNamespace());
+ continue;
+ }
+
+ Attribute timeStampAttr =
attributes.get("uSNChanged");
+ if (timeStampAttr != null) {
+ String uSNChangedVal = (String)
timeStampAttr.get();
+ long currentDeltaSyncTime =
Long.parseLong(uSNChangedVal);
+ LOG.info("uSNChangedVal = {} and
currentDeltaSyncTime = {}", uSNChangedVal, currentDeltaSyncTime);
+ if (currentDeltaSyncTime >
highestdeltaSyncUserTime) {
+ highestdeltaSyncUserTime =
currentDeltaSyncTime;
+ }
+ } else {
+ timeStampAttr =
attributes.get("modifytimestamp");
+ if (timeStampAttr != null) {
+ String timeStampVal = (String)
timeStampAttr.get();
+ Date parseDate =
dateFormat.parse(timeStampVal);
+ long currentDeltaSyncTime =
parseDate.getTime();
+ LOG.info("timeStampVal = {} and
currentDeltaSyncTime = {}", timeStampVal, currentDeltaSyncTime);
+ if (currentDeltaSyncTime >
highestdeltaSyncUserTime) {
+ highestdeltaSyncUserTime =
currentDeltaSyncTime;
+ deltaSyncUserTimeStamp =
timeStampVal;
+ }
+ }
+ }
+
+ // Get all the groups from the group name
attribute of the user only when group search is not enabled.
+ if (!groupSearchEnabled) {
+ for (String useGroupNameAttribute :
userGroupNameAttributeSet) {
+ Attribute userGroupfAttribute =
userEntry.getAttributes().get(useGroupNameAttribute);
+ if (userGroupfAttribute != null) {
+ NamingEnumeration<?> groupEnum =
userGroupfAttribute.getAll();
+ while (groupEnum.hasMore()) {
+ String groupDN = (String)
groupEnum.next();
+ if (LOG.isDebugEnabled()) {
Review Comment:
Consider removing `if (LOG.isDebugEnabled()) {`, unless arguments to log
calls are expensive to compute.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: dev-unsubscr...@ranger.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
