This is an automated email from the ASF dual-hosted git repository.
dhavalshah9131 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push:
new e823f4cf0 RANGER-5427 : AD Groups with 1500+ Users Fail to Sync into
Ranger Admin via RangerUserSync (#966)
e823f4cf0 is described below
commit e823f4cf08961fb1aab66e57fdd0fe59bd23892e
Author: dhavalshah9131 <[email protected]>
AuthorDate: Mon Jul 13 11:27:22 2026 +0530
RANGER-5427 : AD Groups with 1500+ Users Fail to Sync into Ranger Admin via
RangerUserSync (#966)
Co-authored-by: Dhaval Shah <dhavalshah9131@@gmail.com>
---
.../ugsyncutil/util/UgsyncCommonConstants.java | 6 +-
.../ldapusersync/process/LdapUserGroupBuilder.java | 392 ++++++++++++---------
.../unixusersync/config/UserGroupSyncConfig.java | 15 +
.../process/PolicyMgrUserGroupBuilder.java | 3 +-
.../process/TestLdapUserGroupBuilder.java | 272 +++++++++++++-
5 files changed, 517 insertions(+), 171 deletions(-)
diff --git
a/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/util/UgsyncCommonConstants.java
b/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/util/UgsyncCommonConstants.java
index 6f8905a35..c86e2dba3 100644
---
a/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/util/UgsyncCommonConstants.java
+++
b/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/util/UgsyncCommonConstants.java
@@ -22,10 +22,14 @@
public class UgsyncCommonConstants {
public enum CaseConversion { NONE, TO_LOWER, TO_UPPER }
- public static final String ORIGINAL_NAME = "original_name";
+ public static final String ORIGINAL_NAME = "original_name";
public static final String FULL_NAME = "full_name";
public static final String SYNC_SOURCE = "sync_source";
public static final String LDAP_URL = "ldap_url";
+ public static final String PRINCIPAL =
"ranger.usersync.kerberos.principal";
+ public static final String AUTHENTICATION_TYPE =
"hadoop.security.authentication";
+ public static final String KEYTAB =
"ranger.usersync.kerberos.keytab";
+ public static final String NAME_RULE = "hadoop.security.auth_to_local";
public static final String UGSYNC_NONE_CASE_CONVERSION_VALUE = "none";
public static final String UGSYNC_LOWER_CASE_CONVERSION_VALUE = "lower";
diff --git
a/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java
b/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java
index 7f1463ce7..afcd7946b 100644
---
a/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java
+++
b/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java
@@ -21,6 +21,7 @@
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.security.SecureClientLogin;
import org.apache.hadoop.thirdparty.com.google.common.collect.HashBasedTable;
import org.apache.hadoop.thirdparty.com.google.common.collect.Table;
import org.apache.ranger.ugsyncutil.model.LdapSyncSourceInfo;
@@ -49,7 +50,11 @@
import javax.naming.ldap.Rdn;
import javax.naming.ldap.StartTlsRequest;
import javax.naming.ldap.StartTlsResponse;
+import javax.security.auth.Subject;
+import java.io.IOException;
+import java.net.UnknownHostException;
+import java.security.PrivilegedAction;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Arrays;
@@ -70,6 +75,7 @@ public class LdapUserGroupBuilder implements UserGroupSource {
private static final String DATE_FORMAT = "yyyyMMddHHmmss";
private static final String MEMBER_OF_ATTR = "memberof=";
private static final String GROUP_NAME_ATTRIBUTE = "cn=";
+ public static String localHostname = "unknown";
private static final int PAGE_SIZE = 500;
/* for AD uSNChanged */
@@ -98,6 +104,9 @@ public class LdapUserGroupBuilder implements UserGroupSource
{
private String ldapBindDn;
private String ldapBindPassword;
private String ldapAuthenticationMechanism;
+ private String principal;
+ private String keytab;
+ private String nameRules;
private String ldapReferral;
private String searchBase;
private String userNameAttribute;
@@ -280,7 +289,7 @@ public void updateSink(UserGroupSink sink) throws Throwable
{
}
}
- private void createLdapContext() throws Throwable {
+ private void initializeLdapContext() throws Throwable {
Properties env = new Properties();
env.put(Context.INITIAL_CONTEXT_FACTORY,
"com.sun.jndi.ldap.LdapCtxFactory");
@@ -318,7 +327,16 @@ private void createLdapContext() throws Throwable {
}
}
- ldapContext = new InitialLdapContext(env, null);
+ env.put(Context.SECURITY_AUTHENTICATION, ldapAuthenticationMechanism);
+ env.put(Context.SECURITY_PRINCIPAL, ldapBindDn);
+ env.put(Context.SECURITY_CREDENTIALS, ldapBindPassword);
+ env.put(Context.REFERRAL, ldapReferral);
+
+ try {
+ ldapContext = new InitialLdapContext(env, null);
+ } catch (NamingException e) {
+ throw e;
+ }
if (!ldapUrl.startsWith("ldaps")) {
if (config.isStartTlsEnabled()) {
@@ -333,11 +351,22 @@ private void createLdapContext() throws Throwable {
LOG.info("Starting TLS session...");
}
}
+ }
- ldapContext.addToEnvironment(Context.SECURITY_PRINCIPAL, ldapBindDn);
- ldapContext.addToEnvironment(Context.SECURITY_CREDENTIALS,
ldapBindPassword);
- ldapContext.addToEnvironment(Context.SECURITY_AUTHENTICATION,
ldapAuthenticationMechanism);
- ldapContext.addToEnvironment(Context.REFERRAL, ldapReferral);
+ private void createLdapContext() throws Throwable {
+ if (ldapAuthenticationMechanism.equalsIgnoreCase("GSSAPI")) {
+ Subject sub = SecureClientLogin.loginUserFromKeytab(principal,
keytab, nameRules);
+ Subject.doAs(sub, (PrivilegedAction<Void>) () -> {
+ try {
+ initializeLdapContext();
+ } catch (Throwable e) {
+ LOG.error("Failed to create LDAP context: ", e);
+ }
+ return null;
+ });
+ } else {
+ initializeLdapContext();
+ }
}
private void setConfig() throws Throwable {
@@ -351,6 +380,13 @@ private void setConfig() throws Throwable {
ldapBindDn = config.getLdapBindDn();
ldapBindPassword = config.getLdapBindPassword();
ldapAuthenticationMechanism = config.getLdapAuthenticationMechanism();
+ try {
+ principal =
SecureClientLogin.getPrincipal(config.getProperty(UgsyncCommonConstants.PRINCIPAL,
""), localHostname);
+ } catch (IOException ignored) {
+ // do nothing
+ }
+ keytab =
config.getProperty(UgsyncCommonConstants.KEYTAB, "");
+ nameRules =
config.getProperty(UgsyncCommonConstants.NAME_RULE, "DEFAULT");
ldapReferral = config.getContextReferral();
searchBase = config.getSearchBase();
userSearchBase = config.getUserSearchBase().split(";");
@@ -649,28 +685,7 @@ private long getUsers(boolean computeDeletes) throws
Throwable {
}
}
- // Examine the paged results control response
- Control[] controls = ldapContext.getResponseControls();
-
- if (controls != null) {
- for (Control control : controls) {
- if (control instanceof
PagedResultsResponseControl) {
- PagedResultsResponseControl prrc =
(PagedResultsResponseControl) control;
-
- total = prrc.getResultSize();
-
- if (total != 0) {
- LOG.debug("END-OF-PAGE total : {}",
total);
- } else {
- LOG.debug("END-OF-PAGE total :
unknown");
- }
-
- cookie = prrc.getCookie();
- }
- }
- } else {
- LOG.debug("No controls were sent from the server");
- }
+ cookie = handlePagedResults();
// Re-activate paged results
if (pagedResultsEnabled) {
@@ -708,6 +723,7 @@ private long getGroups(boolean computeDeletes) throws
Throwable {
NamingEnumeration<SearchResult> groupSearchResultEnum = null;
DateFormat dateFormat = new
SimpleDateFormat(DATE_FORMAT);
long highestdeltaSyncGroupTime =
deltaSyncGroupTime;
+ Map<String, Set<String>> largeGroups = new
HashMap<>();
try {
createLdapContext();
@@ -741,15 +757,16 @@ private long getGroups(boolean computeDeletes) throws
Throwable {
LOG.info("extendedAllGroupsSearchFilter = {}",
extendedAllGroupsSearchFilter);
- for (String s : groupSearchBase) {
+ for (int ou = 0; ou < groupSearchBase.length; ou++) {
byte[] cookie = null;
int counter = 0;
+ Set<String> largeGroupNames = new HashSet<>();
try {
int paged = 0;
do {
- groupSearchResultEnum = ldapContext.search(s,
extendedAllGroupsSearchFilter, groupSearchControls);
+ groupSearchResultEnum =
ldapContext.search(groupSearchBase[ou], extendedAllGroupsSearchFilter,
groupSearchControls);
while (groupSearchResultEnum.hasMore()) {
final SearchResult groupEntry =
groupSearchResultEnum.next();
@@ -823,71 +840,22 @@ private long getGroups(boolean computeDeletes) throws
Throwable {
int userCount = 0;
if (groupMemberAttr == null ||
groupMemberAttr.size() <= 0) {
- try {
- LOG.info("No members available for {}",
gName);
- } catch (Exception e) {
- throw new RuntimeException(e);
+ if (config.isLargeGroupSyncEnabled()) {
+ groupMemberAttr =
attributes.get("member;range=0-1499");
}
-
- sourceGroupUsers.put(groupFullName, new
HashSet<>());
- continue;
- }
-
- NamingEnumeration<?> userEnum =
groupMemberAttr.getAll();
-
- while (userEnum.hasMore()) {
- String originalUserFullName = (String)
userEnum.next();
-
- if (originalUserFullName == null ||
originalUserFullName.trim().isEmpty()) {
+ if (groupMemberAttr == null ||
groupMemberAttr.size() <= 0) {
+ LOG.info("No members available for {}",
gName);
sourceGroupUsers.put(groupFullName, new
HashSet<>());
continue;
+ } else {
+ largeGroupNames.add(gName);
}
-
- userCount++;
-
- if (!userSearchEnabled) {
- Map<String, String> userAttrMap = new
HashMap<>();
- String userName =
getShortName(originalUserFullName);
-
-
userAttrMap.put(UgsyncCommonConstants.ORIGINAL_NAME, userName);
-
userAttrMap.put(UgsyncCommonConstants.FULL_NAME, originalUserFullName);
-
userAttrMap.put(UgsyncCommonConstants.SYNC_SOURCE, currentSyncSource);
-
userAttrMap.put(UgsyncCommonConstants.LDAP_URL, config.getLdapUrl());
-
- sourceUsers.put(originalUserFullName,
userAttrMap);
-
- LOG.debug("As usersearch is disabled,
adding user {} from group member attribute for group {}", userName, gName);
- }
-
- groupUserTable.put(groupFullName,
originalUserFullName, originalUserFullName);
}
+ userCount = processGroupMembers(groupMemberAttr,
groupFullName);
LOG.info("No. of members in the group {} = {}",
gName, userCount);
}
-
- // Examine the paged results control response
- Control[] controls = ldapContext.getResponseControls();
-
- if (controls != null) {
- for (Control control : controls) {
- if (control instanceof
PagedResultsResponseControl) {
- PagedResultsResponseControl prrc =
(PagedResultsResponseControl) control;
-
- total = prrc.getResultSize();
-
- if (total != 0) {
- LOG.debug("END-OF-PAGE total: {}",
total);
- } else {
- LOG.debug("END-OF-PAGE total:
unknown");
- }
-
- cookie = prrc.getCookie();
- }
- }
- } else {
- LOG.debug("No controls were sent from the server");
- }
-
+ cookie = handlePagedResults();
// Re-activate paged results
if (pagedResultsEnabled) {
LOG.debug("Fetched paged results round: {}",
++paged);
@@ -901,6 +869,10 @@ private long getGroups(boolean computeDeletes) throws
Throwable {
LOG.error("LdapUserGroupBuilder.getGroups() failed with
exception: ", t);
LOG.info("LdapUserGroupBuilder.getGroups() group count:
{}", counter);
}
+ if (config.isLargeGroupSyncEnabled() &&
!largeGroupNames.isEmpty()) {
+ LOG.info("Large groups found: {}", largeGroupNames);
+ largeGroups.put(groupSearchBase[ou], largeGroupNames);
+ }
}
} finally {
if (groupSearchResultEnum != null) {
@@ -908,6 +880,10 @@ private long getGroups(boolean computeDeletes) throws
Throwable {
}
closeLdapContext();
+ if (config.isLargeGroupSyncEnabled() &&
MapUtils.isNotEmpty(largeGroups)) {
+ LOG.info("Large groups found: {}", largeGroups);
+ getRemainingGroupMembers(largeGroups);
+ }
}
if (groupHierarchyLevels > 0) {
@@ -1056,59 +1032,10 @@ private void goUpGroupHierarchyLdap(Set<String>
groupDNs, int groupHierarchyLeve
}
sourceGroups.put(groupFullName, groupAttrMap);
-
- NamingEnumeration<?> userEnum =
groupMemberAttr.getAll();
-
- while (userEnum.hasMore()) {
- String originalUserFullName = (String)
userEnum.next();
-
- if (originalUserFullName == null ||
originalUserFullName.trim().isEmpty()) {
- continue;
- }
-
- userCount++;
-
- if (!userSearchEnabled &&
!sourceGroups.containsKey(originalUserFullName)) {
- Map<String, String> userAttrMap = new
HashMap<>();
- String userName =
getShortName(originalUserFullName);
-
-
userAttrMap.put(UgsyncCommonConstants.ORIGINAL_NAME, userName);
-
userAttrMap.put(UgsyncCommonConstants.FULL_NAME, originalUserFullName);
-
userAttrMap.put(UgsyncCommonConstants.SYNC_SOURCE, currentSyncSource);
-
userAttrMap.put(UgsyncCommonConstants.LDAP_URL, config.getLdapUrl());
-
- sourceUsers.put(originalUserFullName,
userAttrMap);
- }
-
- groupUserTable.put(groupFullName,
originalUserFullName, originalUserFullName);
- }
-
+ userCount = processGroupMembers(groupMemberAttr,
groupFullName);
LOG.info("No. of members in the group {} = {}",
gName, userCount);
}
-
- // Examine the paged results control response
- Control[] controls = ldapContext.getResponseControls();
-
- if (controls != null) {
- for (Control control : controls) {
- if (control instanceof
PagedResultsResponseControl) {
- PagedResultsResponseControl prrc =
(PagedResultsResponseControl) control;
-
- total = prrc.getResultSize();
-
- if (total != 0) {
- LOG.debug("END-OF-PAGE total : {}",
total);
- } else {
- LOG.debug("END-OF-PAGE total :
unknown");
- }
-
- cookie = prrc.getCookie();
- }
- }
- } else {
- LOG.debug("No controls were sent from the server");
- }
-
+ cookie = handlePagedResults();
// Re-activate paged results
if (pagedResultsEnabled) {
ldapContext.setRequestControls(new Control[] {new
PagedResultsControl(pagedResultsSize, cookie, Control.CRITICAL)});
@@ -1149,7 +1076,7 @@ private void addToAttrMap(Map<String, String>
userAttrMap, String attrName, Attr
} catch (ClassCastException e) {
LOG.error("{} type is not set properly {}", attrName,
e.getMessage());
}
- } else if (attrType.equals("String")) {
+ } else if (attrType.equalsIgnoreCase("String")) {
userAttrMap.put(attrName, (String) attr.get());
} else {
LOG.warn("Attribute Type {} not supported for {}", attrType,
attrName);
@@ -1208,6 +1135,160 @@ private static String getShortName(String longName) {
return shortName;
}
+ private byte[] handlePagedResults() throws Throwable {
+ Control[] controls = ldapContext.getResponseControls();
+
+ if (controls != null) {
+ for (int i = 0; i < controls.length; i++) {
+ if (controls[i] instanceof PagedResultsResponseControl) {
+ PagedResultsResponseControl prrc =
(PagedResultsResponseControl) controls[i];
+ LOG.debug("END-OF-PAGE total: {}", prrc.getResultSize());
+ return prrc.getCookie();
+ }
+ }
+ } else {
+ LOG.debug("No controls were sent from the server");
+ }
+ return null;
+ }
+
+ private int processGroupMembers(Attribute groupMemberAttr, String
groupFullName) throws Throwable {
+ NamingEnumeration<?> userEnum = groupMemberAttr.getAll();
+ int userCount = 0;
+ while (userEnum.hasMore()) {
+ String originalUserFullName = (String) userEnum.next();
+
+ if (originalUserFullName == null ||
originalUserFullName.trim().isEmpty()) {
+ sourceGroupUsers.put(groupFullName, new HashSet<>());
+ continue;
+ }
+
+ userCount++;
+
+ if (!userSearchEnabled) {
+ Map<String, String> userAttrMap = new HashMap<>();
+ String userName =
getShortName(originalUserFullName);
+
+ userAttrMap.put(UgsyncCommonConstants.ORIGINAL_NAME, userName);
+ userAttrMap.put(UgsyncCommonConstants.FULL_NAME,
originalUserFullName);
+ userAttrMap.put(UgsyncCommonConstants.SYNC_SOURCE,
currentSyncSource);
+ userAttrMap.put(UgsyncCommonConstants.LDAP_URL,
config.getLdapUrl());
+
+ sourceUsers.put(originalUserFullName, userAttrMap);
+
+ LOG.debug("As usersearch is disabled, adding user {} from
group member attribute for group {}", userName, groupFullName);
+ }
+
+ groupUserTable.put(groupFullName, originalUserFullName,
originalUserFullName);
+ }
+ return userCount;
+ }
+
+ private void getRemainingGroupMembers(Map<String, Set<String>>
largeGroups) throws Throwable {
+ NamingEnumeration<SearchResult> groupSearchResultEnum = null;
+
+ try {
+ createLdapContext();
+
+ int counter = 0;
+
+ // Loop through each OU
+ for (String ou : largeGroups.keySet()) {
+ Set<String> groupNames = largeGroups.get(ou);
+
+ // Process each group individually
+ for (String groupName : groupNames) {
+ LOG.info("Processing large group: {}", groupName);
+
+ // Build filter for THIS group only
+ String groupFilter = "(&(objectclass=" + groupObjectClass
+ ")";
+
+ // Add custom filter if present
+ if (groupSearchFilter != null &&
!groupSearchFilter.trim().isEmpty()) {
+ String customFilter = groupSearchFilter.trim();
+ if (!customFilter.startsWith("(")) {
+ customFilter = "(" + customFilter + ")";
+ }
+ groupFilter += customFilter;
+ }
+
+ // Add the specific group name
+ groupFilter += "(CN=" + groupName + "))";
+
+ LOG.info("Group filter for {}: {}", groupName,
groupFilter);
+
+ try {
+ int rangeStep = 1500;
+ int start = 1500;
+ boolean lastRange = false;
+
+ // Loop until we get all members for THIS group
+ while (!lastRange) {
+ // Build the range attribute name
+ String rangeAttr = "member;range=" + start +
"-" + (start + rangeStep - 1);
+ String[] attributes = {rangeAttr};
+
+ LOG.info("Fetching range for {}: {}", groupName,
rangeAttr);
+
+ // Prepare search controls
+ SearchControls controls = new SearchControls();
+
controls.setSearchScope(SearchControls.SUBTREE_SCOPE);
+ controls.setReturningAttributes(attributes);
+
+ // Search for this specific group
+ groupSearchResultEnum = ldapContext.search(ou,
groupFilter, controls);
+
+ while (groupSearchResultEnum.hasMore()) {
+ final SearchResult result =
groupSearchResultEnum.next();
+ Attributes attrs =
result.getAttributes();
+ String groupFullName =
result.getNameInNamespace();
+
+ LOG.info("Processing range for group: {}",
groupFullName);
+
+ NamingEnumeration<? extends Attribute>
attrEnum = attrs.getAll();
+
+ while (attrEnum.hasMore()) {
+ Attribute attr = attrEnum.next();
+ String attrID = attr.getID();
+
+ LOG.info("Attribute ID: {}", attrID);
+
+ // Check if this is the last range for
THIS group
+ if (attrID.endsWith("*")) {
+ lastRange = true;
+ LOG.info("Last range reached for
group: {}", groupName);
+ }
+
+ int userCount = processGroupMembers(attr,
groupFullName);
+ LOG.info("No. of members in the group {}
for range {} = {}", groupFullName, rangeAttr, userCount);
+ }
+
+ counter++;
+ }
+
+ // Move to next range
+ start += rangeStep;
+ }
+
+ LOG.info("Completed processing group: {} with total
iterations: {}", groupName, counter);
+ } catch (Exception e) {
+ LOG.error("Error processing group {}: {}", groupName,
e.getMessage(), e);
+ }
+ }
+ }
+
+ LOG.info("Completed processing all large groups");
+ } catch (Throwable t) {
+ LOG.error("LdapUserGroupBuilder.getRemainingGroupMembers() failed
with exception:", t);
+ throw t;
+ } finally {
+ if (groupSearchResultEnum != null) {
+ groupSearchResultEnum.close();
+ }
+ closeLdapContext();
+ }
+ }
+
private String getFirstRDN(String name) {
if (StringUtils.isEmpty(name)) {
return null;
@@ -1333,30 +1414,7 @@ private String getDNForMemberOf(String searchFilter)
throws Throwable {
}
}
}
-
- // Examine the paged results control response
- Control[] controls = ldapContext.getResponseControls();
-
- if (controls != null) {
- for (Control control : controls) {
- if (control instanceof
PagedResultsResponseControl) {
- PagedResultsResponseControl prrc =
(PagedResultsResponseControl) control;
-
- total = prrc.getResultSize();
-
- if (total != 0) {
- LOG.debug("END-OF-PAGE total : {}",
total);
- } else {
- LOG.debug("END-OF-PAGE total :
unknown");
- }
-
- cookie = prrc.getCookie();
- }
- }
- } else {
- LOG.debug("No controls were sent from the server");
- }
-
+ cookie = handlePagedResults();
// Re-activate paged results
if (pagedResultsEnabled) {
LOG.debug("Fetched paged results round: {}",
++paged);
@@ -1384,4 +1442,12 @@ private String getDNForMemberOf(String searchFilter)
throws Throwable {
return computedSearchFilter;
}
+
+ static {
+ try {
+ localHostname =
java.net.InetAddress.getLocalHost().getCanonicalHostName();
+ } catch (UnknownHostException e) {
+ localHostname = "unknown";
+ }
+ }
}
diff --git
a/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java
b/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java
index 08ca72500..eb501ea12 100644
---
a/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java
+++
b/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java
@@ -137,6 +137,8 @@ public class UserGroupSyncConfig {
private static final int DEFAULT_LGSYNC_GROUP_HIERARCHY_LEVELS
= 0;
private static final int DEFAULT_LGSYNC_PAGED_RESULTS_SIZE
= 500;
private static final boolean DEFAULT_LGSYNC_LDAP_DELTASYNC_ENABLED
= false;
+ private static final String LGSYNC_LDAP_LARGEGROUPSYNC_ENABLED
= "ranger.usersync.ldap.largegroupsync";
+ private static final boolean DEFAULT_LGSYNC_LDAP_LARGEGROUPSYNC_ENABLED
= false;
private static final boolean DEFAULT_LGSYNC_LDAP_STARTTLS_ENABLED
= false;
private static final boolean DEFAULT_LGSYNC_PAGED_RESULTS_ENABLED
= true;
private static final boolean DEFAULT_LGSYNC_GROUP_SEARCH_ENABLED
= true;
@@ -1389,6 +1391,19 @@ public boolean isSyncSourceValidationEnabled() {
return isSyncSourceValidationEnabled;
}
+ public boolean isLargeGroupSyncEnabled() {
+ boolean largeGroupSyncEnabled;
+ String val = prop.getProperty(LGSYNC_LDAP_LARGEGROUPSYNC_ENABLED);
+
+ if (val == null || val.trim().isEmpty()) {
+ largeGroupSyncEnabled = DEFAULT_LGSYNC_LDAP_LARGEGROUPSYNC_ENABLED;
+ } else {
+ largeGroupSyncEnabled = Boolean.valueOf(val);
+ }
+
+ return largeGroupSyncEnabled;
+ }
+
private void init() {
XMLUtils.loadConfig(DEFAULT_CONFIG_FILE, prop);
XMLUtils.loadConfig(CORE_SITE_CONFIG_FILE, prop);
diff --git
a/ugsync/src/main/java/org/apache/ranger/unixusersync/process/PolicyMgrUserGroupBuilder.java
b/ugsync/src/main/java/org/apache/ranger/unixusersync/process/PolicyMgrUserGroupBuilder.java
index ad8170efd..b42700cd8 100644
---
a/ugsync/src/main/java/org/apache/ranger/unixusersync/process/PolicyMgrUserGroupBuilder.java
+++
b/ugsync/src/main/java/org/apache/ranger/unixusersync/process/PolicyMgrUserGroupBuilder.java
@@ -81,7 +81,6 @@ public class PolicyMgrUserGroupBuilder extends
AbstractUserGroupSource implement
public static final String PM_UPDATE_USERS_ROLES_URI =
"/service/xusers/users/roleassignments";
/* ******************* */
- private static final String AUTHENTICATION_TYPE =
"hadoop.security.authentication";
private static final String AUTH_KERBEROS =
"kerberos";
private static final String KERBEROS_PRINCIPAL =
"ranger.usersync.kerberos.principal";
private static final String KERBEROS_KEYTAB =
"ranger.usersync.kerberos.keytab";
@@ -205,7 +204,7 @@ public synchronized void init() throws Throwable {
String keyStoreType = config.getSSLKeyStoreType();
String trustStoreType = config.getSSLTrustStoreType();
- authenticationType = config.getProperty(AUTHENTICATION_TYPE, "simple");
+ authenticationType =
config.getProperty(UgsyncCommonConstants.AUTHENTICATION_TYPE, "simple");
try {
principal =
SecureClientLogin.getPrincipal(config.getProperty(KERBEROS_PRINCIPAL, ""),
localHostname);
diff --git
a/ugsync/src/test/java/org/apache/ranger/ldapusersync/process/TestLdapUserGroupBuilder.java
b/ugsync/src/test/java/org/apache/ranger/ldapusersync/process/TestLdapUserGroupBuilder.java
index 0b9624390..4c2af85af 100644
---
a/ugsync/src/test/java/org/apache/ranger/ldapusersync/process/TestLdapUserGroupBuilder.java
+++
b/ugsync/src/test/java/org/apache/ranger/ldapusersync/process/TestLdapUserGroupBuilder.java
@@ -31,15 +31,19 @@
import org.apache.ranger.ugsyncutil.model.UgsyncAuditInfo;
import org.apache.ranger.unixusersync.config.UserGroupSyncConfig;
import org.apache.ranger.usergroupsync.UserGroupSink;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.MethodOrderer;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestMethodOrder;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.MockedConstruction;
+import org.mockito.MockedStatic;
import org.mockito.Mockito;
import org.mockito.junit.jupiter.MockitoExtension;
import javax.naming.NamingEnumeration;
+import javax.naming.NamingException;
import javax.naming.directory.Attribute;
import javax.naming.directory.Attributes;
import javax.naming.directory.BasicAttribute;
@@ -87,7 +91,8 @@
@CreateLdapConnectionPool(maxActive = 1, maxWait = 5000)
@ApplyLdifFiles("ADSchema.ldif")
public class TestLdapUserGroupBuilder extends AbstractLdapTestUnit {
- private static final String STR_EPOCH = new
SimpleDateFormat("yyyyMMddHHmmss").format(new Date(0));
+ private static final String STR_EPOCH = new
SimpleDateFormat("yyyyMMddHHmmss").format(new Date(0));
+ private MockedStatic<UserGroupSyncConfig> mockedConfig;
@Test
public void testA_init_and_isChanged() throws Exception {
@@ -530,6 +535,7 @@ public void
testW_getGroups_processes_groups_and_memberships() throws Throwable
configureMinimalLdapConfig();
UserGroupSyncConfig cfg = UserGroupSyncConfig.getInstance();
cfg.setProperty("ranger.usersync.ldap.group.searchfilter",
"cn=Group*");
+ cfg.setProperty("ranger.usersync.ldap.largegroupsync", "true");
LdapUserGroupBuilder b = new LdapUserGroupBuilder();
b.init();
@@ -667,7 +673,7 @@ public void
testX_goUpGroupHierarchyLdap_fetches_nested_groups_and_members() thr
configureMinimalLdapConfig();
UserGroupSyncConfig cfg = UserGroupSyncConfig.getInstance();
cfg.setProperty("ranger.usersync.ldap.group.searchfilter",
"cn=Nested*");
-
+ cfg.setProperty("ranger.usersync.ldap.groupattributelist", "");
LdapUserGroupBuilder b = new LdapUserGroupBuilder();
b.init();
@@ -677,15 +683,18 @@ public void
testX_goUpGroupHierarchyLdap_fetches_nested_groups_and_members() thr
fUserEnabled.setBoolean(b, false);
// Prepare internal structures used by goUpGroupHierarchyLdap
- Field gutF =
LdapUserGroupBuilder.class.getDeclaredField("groupUserTable");
- Field srcUsersF =
LdapUserGroupBuilder.class.getDeclaredField("sourceUsers");
- Field srcGroupsF =
LdapUserGroupBuilder.class.getDeclaredField("sourceGroups");
+ Field gutF =
LdapUserGroupBuilder.class.getDeclaredField("groupUserTable");
+ Field srcUsersF =
LdapUserGroupBuilder.class.getDeclaredField("sourceUsers");
+ Field srcGroupsF =
LdapUserGroupBuilder.class.getDeclaredField("sourceGroups");
+ Field srcGroupUsersF =
LdapUserGroupBuilder.class.getDeclaredField("sourceGroupUsers");
gutF.setAccessible(true);
srcUsersF.setAccessible(true);
srcGroupsF.setAccessible(true);
+ srcGroupUsersF.setAccessible(true);
gutF.set(b, HashBasedTable.create());
srcUsersF.set(b, new HashMap<>());
srcGroupsF.set(b, new HashMap<>());
+ srcGroupUsersF.set(b, new HashMap<>());
// Parent group DN used to build filter
final String parentGroupDn =
"CN=Parent,OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com";
@@ -867,6 +876,259 @@ public SearchResult nextElement() {
}
}
+ @BeforeEach
+ void setupActiveState() {
+ // Mock the static method to always return true for active state
+ mockedConfig = Mockito.mockStatic(UserGroupSyncConfig.class,
Mockito.CALLS_REAL_METHODS);
+
mockedConfig.when(UserGroupSyncConfig::isUgsyncServiceActive).thenReturn(true);
+ }
+
+ @AfterEach
+ void tearDownActiveState() {
+ if (mockedConfig != null) {
+ mockedConfig.close();
+ }
+ }
+
+ @Test
+ void testZ_largeGroupSyncEnabled_detectsInitialRange() throws Throwable {
+ resetConfig();
+ configureMinimalLdapConfig();
+ UserGroupSyncConfig cfg = UserGroupSyncConfig.getInstance();
+ // FIX: Use the correct property name
+ cfg.setProperty("ranger.usersync.ldap.largegroupsync", "true");
+
+ LdapUserGroupBuilder b = new LdapUserGroupBuilder();
+ b.init();
+
+ // Verify that large group sync is enabled in config
+ assertTrue(cfg.isLargeGroupSyncEnabled(), "Large group sync should be
enabled");
+
+ // Verify the processGroupMembers method exists
+ Method processGroupMembers =
LdapUserGroupBuilder.class.getDeclaredMethod("processGroupMembers",
Attribute.class, String.class);
+ assertNotNull(processGroupMembers, "processGroupMembers method should
exist");
+
+ // Verify getRemainingGroupMembers method exists (new method for large
groups)
+ Method getRemainingGroupMembers =
LdapUserGroupBuilder.class.getDeclaredMethod("getRemainingGroupMembers",
Map.class);
+ assertNotNull(getRemainingGroupMembers, "getRemainingGroupMembers
method should exist");
+
+ // Verify that group search controls include the ranged attribute when
large group sync is enabled
+ Field groupSearchControlsF =
LdapUserGroupBuilder.class.getDeclaredField("groupSearchControls");
+ groupSearchControlsF.setAccessible(true);
+ SearchControls controls = (SearchControls)
groupSearchControlsF.get(b);
+ String[] returningAttrs = controls.getReturningAttributes();
+
+ // Check if "member;range=0-1499" is in the returning attributes
+ boolean hasRangeAttr = false;
+ for (String attr : returningAttrs) {
+ if (attr.equals("member;range=0-1499")) {
+ hasRangeAttr = true;
+ break;
+ }
+ }
+ assertFalse(hasRangeAttr, "Should NOT include member;range=0-1499 in
initial search - it prevents AD from returning regular member attribute for
small groups");
+ }
+
+ @Test
+ void testZA_getRemainingGroupMembers_handles_ranged_attributes() throws
Throwable {
+ resetConfig();
+ configureMinimalLdapConfig();
+ UserGroupSyncConfig cfg = UserGroupSyncConfig.getInstance();
+ cfg.setProperty("ranger.usersync.ldap.largegroupsync", "true");
+
+ LdapUserGroupBuilder b = new LdapUserGroupBuilder();
+ b.init();
+
+ // Prepare internal structures
+ Field gutF =
LdapUserGroupBuilder.class.getDeclaredField("groupUserTable");
+ Field srcUsersF =
LdapUserGroupBuilder.class.getDeclaredField("sourceUsers");
+ Field srcGroupsF =
LdapUserGroupBuilder.class.getDeclaredField("sourceGroups");
+ gutF.setAccessible(true);
+ srcUsersF.setAccessible(true);
+ srcGroupsF.setAccessible(true);
+ gutF.set(b, HashBasedTable.create());
+ srcUsersF.set(b, new HashMap<>());
+ srcGroupsF.set(b, new HashMap<>());
+
+ Method getRemainingMembers =
LdapUserGroupBuilder.class.getDeclaredMethod("getRemainingGroupMembers",
Map.class);
+ getRemainingMembers.setAccessible(true);
+
+ // This will throw because it tries to create LDAP context
+ // Test that it throws Throwable (which is expected without a real
LDAP server)
+ Map<String, Set<String>> emptyMap = new HashMap<>();
+ assertThrows(Throwable.class, () -> getRemainingMembers.invoke(b,
emptyMap));
+ }
+
+ @Test
+ void testZB_processGroupMembers_handles_empty_members() throws Throwable {
+ resetConfig();
+ configureMinimalLdapConfig();
+
+ LdapUserGroupBuilder b = new LdapUserGroupBuilder();
+ b.init();
+
+ // Prepare internal structures
+ Field gutF =
LdapUserGroupBuilder.class.getDeclaredField("groupUserTable");
+ Field srcGroupUsersF =
LdapUserGroupBuilder.class.getDeclaredField("sourceGroupUsers");
+ gutF.setAccessible(true);
+ srcGroupUsersF.setAccessible(true);
+ gutF.set(b, HashBasedTable.create());
+ srcGroupUsersF.set(b, new HashMap<>());
+
+ Method processGroupMembers =
LdapUserGroupBuilder.class.getDeclaredMethod("processGroupMembers",
Attribute.class, String.class);
+ processGroupMembers.setAccessible(true);
+
+ final String groupDn = "CN=TestGroup,OU=Groups,DC=example,DC=com";
+
+ // Create attribute with mix of empty and valid members
+ BasicAttribute members = new BasicAttribute("member");
+ members.add(""); // empty member - causes sourceGroupUsers.put and
continue
+ members.add("uid=validuser,ou=people,dc=example,dc=com");
+
+ int count = (Integer) processGroupMembers.invoke(b, members, groupDn);
+
+ // Only the valid member is counted (empty member triggers continue
before userCount++)
+ assertEquals(1, count);
+
+ @SuppressWarnings("unchecked")
+ Table<String, String, String> gut = (Table<String, String, String>)
gutF.get(b);
+
+ // Only valid user should be in the table
+ assertFalse(gut.contains(groupDn, ""));
+ assertTrue(gut.contains(groupDn,
"uid=validuser,ou=people,dc=example,dc=com"));
+
+ // sourceGroupUsers should have been set (from the empty member check)
+ @SuppressWarnings("unchecked")
+ Map<String, Set<String>> srcGroupUsers = (Map<String, Set<String>>)
srcGroupUsersF.get(b);
+ assertTrue(srcGroupUsers.containsKey(groupDn));
+ }
+
+ @Test
+ void testZC_largeGroupSync_disabled_skips_remaining_members() throws
Throwable {
+ resetConfig();
+ configureMinimalLdapConfig();
+ UserGroupSyncConfig cfg = UserGroupSyncConfig.getInstance();
+ cfg.setProperty("ranger.usersync.ldap.group.searchfilter",
"cn=LargeGroup*");
+ cfg.setProperty("ranger.usersync.large.group.sync.enabled", "false");
+
+ LdapUserGroupBuilder b = new LdapUserGroupBuilder();
+ b.init();
+
+ // Verify that large group sync is disabled
+ assertFalse(cfg.isLargeGroupSyncEnabled());
+
+ // Test that with disabled large group sync, processGroupMembers still
works normally
+ Field gutF =
LdapUserGroupBuilder.class.getDeclaredField("groupUserTable");
+ Field srcGroupUsersF =
LdapUserGroupBuilder.class.getDeclaredField("sourceGroupUsers");
+ gutF.setAccessible(true);
+ srcGroupUsersF.setAccessible(true);
+ gutF.set(b, HashBasedTable.create());
+ srcGroupUsersF.set(b, new HashMap<>());
+
+ Method processGroupMembers =
LdapUserGroupBuilder.class.getDeclaredMethod("processGroupMembers",
Attribute.class, String.class);
+ processGroupMembers.setAccessible(true);
+
+ final String groupDn =
"CN=NormalGroup,OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com";
+ BasicAttribute members = new BasicAttribute("member");
+ members.add("uid=user1,ou=people,dc=example,dc=com");
+ members.add("uid=user2,ou=people,dc=example,dc=com");
+
+ int count = (Integer) processGroupMembers.invoke(b, members, groupDn);
+ assertEquals(2, count);
+ }
+
+ @Test
+ void testZD_getRemainingGroupMembers_handles_exceptions() throws Throwable
{
+ resetConfig();
+ configureMinimalLdapConfig();
+
+ LdapUserGroupBuilder b = new LdapUserGroupBuilder();
+ b.init();
+
+ // Prepare internal structures
+ Field gutF =
LdapUserGroupBuilder.class.getDeclaredField("groupUserTable");
+ gutF.setAccessible(true);
+ gutF.set(b, HashBasedTable.create());
+
+ try (MockedConstruction<InitialLdapContext> mocked =
Mockito.mockConstruction(InitialLdapContext.class, (mock, ctx) -> {
+ Mockito.when(mock.search(Mockito.anyString(), Mockito.anyString(),
Mockito.any(SearchControls.class)))
+ .thenThrow(new NamingException("Simulated LDAP error"));
+ Mockito.when(mock.getResponseControls()).thenReturn(null);
+ })) {
+ Map<String, Set<String>> largeGroups = new HashMap<>();
+ Set<String> groupNames = new HashSet<>();
+ groupNames.add("FailingGroup");
+ largeGroups.put("OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com",
groupNames);
+
+ Method getRemainingMembers =
LdapUserGroupBuilder.class.getDeclaredMethod("getRemainingGroupMembers",
Map.class);
+ getRemainingMembers.setAccessible(true);
+
+ // Should not throw, but log error and continue
+ assertDoesNotThrow(() -> getRemainingMembers.invoke(b,
largeGroups));
+ }
+ }
+
+ @Test
+ void testZE_processGroupMembers_handles_members_correctly() throws
Throwable {
+ resetConfig();
+ configureMinimalLdapConfig();
+
+ LdapUserGroupBuilder b = new LdapUserGroupBuilder();
+ b.init();
+
+ // Disable user search so members are added to sourceUsers directly
+ Field fUserEnabled =
LdapUserGroupBuilder.class.getDeclaredField("userSearchEnabled");
+ Field currentSyncSourceF =
LdapUserGroupBuilder.class.getDeclaredField("currentSyncSource");
+ fUserEnabled.setAccessible(true);
+ currentSyncSourceF.setAccessible(true);
+ fUserEnabled.setBoolean(b, false);
+
+ // Ensure currentSyncSource is set
+ if (currentSyncSourceF.get(b) == null) {
+ currentSyncSourceF.set(b, "LDAP");
+ }
+
+ // Prepare internal structures
+ Field gutF =
LdapUserGroupBuilder.class.getDeclaredField("groupUserTable");
+ Field srcUsersF =
LdapUserGroupBuilder.class.getDeclaredField("sourceUsers");
+ Field srcGroupUsersF =
LdapUserGroupBuilder.class.getDeclaredField("sourceGroupUsers");
+ gutF.setAccessible(true);
+ srcUsersF.setAccessible(true);
+ srcGroupUsersF.setAccessible(true);
+ gutF.set(b, HashBasedTable.create());
+ srcUsersF.set(b, new HashMap<>());
+ srcGroupUsersF.set(b, new HashMap<>());
+
+ Method processGroupMembers =
LdapUserGroupBuilder.class.getDeclaredMethod("processGroupMembers",
Attribute.class, String.class);
+ processGroupMembers.setAccessible(true);
+
+ final String groupDn = "CN=TestGroup,OU=Groups,DC=example,DC=com";
+
+ // Test with a mix of valid and edge case members
+ BasicAttribute members = new BasicAttribute("member");
+ members.add("uid=user1,ou=people,dc=example,dc=com");
+ members.add("uid=user2,ou=people,dc=example,dc=com");
+ members.add("uid=user3,ou=people,dc=example,dc=com");
+
+ int count = (Integer) processGroupMembers.invoke(b, members, groupDn);
+ assertEquals(3, count);
+
+ @SuppressWarnings("unchecked")
+ Table<String, String, String> gut = (Table<String, String, String>)
gutF.get(b);
+
+ // All valid users should be in the table
+ assertTrue(gut.contains(groupDn,
"uid=user1,ou=people,dc=example,dc=com"));
+ assertTrue(gut.contains(groupDn,
"uid=user2,ou=people,dc=example,dc=com"));
+ assertTrue(gut.contains(groupDn,
"uid=user3,ou=people,dc=example,dc=com"));
+
+ // When userSearchEnabled is false, users should also be added to
sourceUsers
+ @SuppressWarnings("unchecked")
+ Map<String, Map<String, String>> srcUsers = (Map<String, Map<String,
String>>) srcUsersF.get(b);
+
assertTrue(srcUsers.containsKey("uid=user1,ou=people,dc=example,dc=com"));
+
assertTrue(srcUsers.containsKey("uid=user2,ou=people,dc=example,dc=com"));
+
assertTrue(srcUsers.containsKey("uid=user3,ou=people,dc=example,dc=com"));
+ }
+
private static void configureMinimalLdapConfig() throws Exception {
UserGroupSyncConfig cfg = UserGroupSyncConfig.getInstance();
cfg.setProperty("ranger.usersync.ldap.url", "ldap://127.0.0.1:389";);
