This is an automated email from the ASF dual-hosted git repository. pradeep pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ranger.git
commit f45054d1b933b5e65cb19ace014d3b2cb0d78132 Author: RickyMa <[email protected]> AuthorDate: Tue Jun 2 13:50:48 2020 +0530 RANGER-2789: GET API service/xusers/users turns very slow when there are more than 1000 users Signed-off-by: pradeep <[email protected]> --- .../main/java/org/apache/ranger/entity/XXUser.java | 6 + .../org/apache/ranger/service/XUserService.java | 141 ++++++++++++++++++++- .../apache/ranger/service/XUserServiceBase.java | 130 +++++++++++++++++++ .../main/java/org/apache/ranger/view/VXUser.java | 19 +++ 4 files changed, 295 insertions(+), 1 deletion(-) diff --git a/security-admin/src/main/java/org/apache/ranger/entity/XXUser.java b/security-admin/src/main/java/org/apache/ranger/entity/XXUser.java index 0464e7b..14f0b64 100644 --- a/security-admin/src/main/java/org/apache/ranger/entity/XXUser.java +++ b/security-admin/src/main/java/org/apache/ranger/entity/XXUser.java @@ -37,6 +37,8 @@ import org.apache.ranger.common.AppConstants; import org.apache.ranger.common.RangerCommonEnums; import org.apache.ranger.common.RangerConstants; +import java.util.Objects; + @Entity @Table(name="x_user") @@ -285,4 +287,8 @@ public class XXUser extends XXDBBase implements java.io.Serializable { return null; } + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), name, description, status, credStoreId); + } } diff --git a/security-admin/src/main/java/org/apache/ranger/service/XUserService.java b/security-admin/src/main/java/org/apache/ranger/service/XUserService.java index 6ff8823..75aa006 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/XUserService.java +++ b/security-admin/src/main/java/org/apache/ranger/service/XUserService.java @@ -25,14 +25,20 @@ import java.util.HashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.MapUtils; import org.apache.ranger.biz.RangerBizUtil; import org.apache.ranger.common.AppConstants; import org.apache.ranger.common.MessageEnums; import org.apache.ranger.common.PropertiesUtil; import org.apache.ranger.common.RangerCommonEnums; import org.apache.ranger.common.RangerConstants; +import org.apache.ranger.common.SearchCriteria; import org.apache.ranger.common.SearchField; import org.apache.ranger.common.SortField; import org.apache.ranger.common.StringUtil; @@ -45,10 +51,13 @@ import org.apache.ranger.entity.XXUser; import org.apache.ranger.util.RangerEnumUtil; import org.apache.ranger.view.VXPortalUser; import org.apache.ranger.view.VXUser; +import org.apache.ranger.view.VXUserList; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Service; -import org.springframework.util.CollectionUtils; + +import static java.util.function.Function.identity; +import static java.util.stream.Collectors.toMap; @Service @Scope("singleton") @@ -207,6 +216,19 @@ public class XUserService extends XUserServiceBase<XXUser, VXUser> { } @Override + protected List<VXUser> mapEntityToViewBeans(Map<VXUser, XXUser> vxUserXXUserMap) { + if (MapUtils.isNotEmpty(vxUserXXUserMap)) { + Map<String, VXUser> userNameVXUserMap = vxUserXXUserMap.keySet() + .stream() + .collect(toMap(VXUser::getName, identity())); + super.mapEntityToViewBeans(vxUserXXUserMap); + populateUserListAttributes(userNameVXUserMap); + return new ArrayList<>(vxUserXXUserMap.keySet()); + } + return new ArrayList<>(); + } + + @Override public VXUser populateViewBean(XXUser xUser) { VXUser vObj = super.populateViewBean(xUser); vObj.setIsVisible(xUser.getIsVisible()); @@ -214,6 +236,38 @@ public class XUserService extends XUserServiceBase<XXUser, VXUser> { return vObj; } + @Override + public List<VXUser> populateViewBeans(List<XXUser> xUsers) { + List<VXUser> vObjList = super.populateViewBeans(xUsers); + if (CollectionUtils.isNotEmpty(vObjList) && CollectionUtils.isNotEmpty(xUsers) && xUsers.size() == vObjList.size()) { + Map<Long, VXUser> xUserIdVObjMap = new HashMap<>(xUsers.size()); + for (int i = 0; i < xUsers.size(); ++i) { + VXUser vObj = vObjList.get(i); + XXUser xUser = xUsers.get(i); + vObj.setIsVisible(xUser.getIsVisible()); + xUserIdVObjMap.put(xUser.getId(), vObj); + } + populateGroupList(xUserIdVObjMap); + } + return vObjList; + } + + /** + * @param searchCriteria + * @return + */ + @Override + public VXUserList searchXUsers(SearchCriteria searchCriteria) { + VXUserList returnList = new VXUserList(); + + @SuppressWarnings("unchecked") + List<XXUser> resultList = searchResources(searchCriteria, + searchFields, sortFields, returnList); + + returnList.setVXUsers(populateViewBeans(resultList)); + return returnList; + } + private void populateGroupList(Long xUserId, VXUser vObj) { List<XXGroupUser> xGroupUserList = daoManager.getXXGroupUser() .findByUserId(xUserId); @@ -231,6 +285,40 @@ public class XUserService extends XUserServiceBase<XXUser, VXUser> { vObj.setGroupNameList(groupNames); } + private void populateGroupList(Map<Long, VXUser> xUserIdVObjMap) { + List<XXGroupUser> allXXGroupUsers = daoManager.getXXGroupUser().getAll(); + if (MapUtils.isNotEmpty(xUserIdVObjMap) && CollectionUtils.isNotEmpty(allXXGroupUsers)) { + Map<Long, List<XXGroupUser>> userIdXXGroupUserMap = new HashMap<>(xUserIdVObjMap.size()); + for (Map.Entry<Long, VXUser> xUserIdVXUserEntry : xUserIdVObjMap.entrySet()) { + Long xUserId = xUserIdVXUserEntry.getKey(); + List<XXGroupUser> xxGroupUsers = allXXGroupUsers + .stream() + .filter(xXGroupUser -> Objects.equals(xXGroupUser.getUserId(), xUserId)) + .collect(Collectors.toList()); + userIdXXGroupUserMap.put(xUserId, xxGroupUsers); + } + for (Map.Entry<Long, List<XXGroupUser>> xUserIdXXGroupUserListEntry : userIdXXGroupUserMap.entrySet()) { + Long xUserId = xUserIdXXGroupUserListEntry.getKey(); + List<XXGroupUser> xGroupUserList = xUserIdXXGroupUserListEntry.getValue(); + Set<Long> groupIdList = new LinkedHashSet<>(); + Set<String> groupNameList = new LinkedHashSet<>(); + if (xGroupUserList != null) { + for (XXGroupUser xGroupUser : xGroupUserList) { + groupIdList.add(xGroupUser.getParentGroupId()); + groupNameList.add(xGroupUser.getName()); + } + } + List<Long> groups = new ArrayList<>(groupIdList); + List<String> groupNames = new ArrayList<>(groupNameList); + VXUser vObj = xUserIdVObjMap.get(xUserId); + if (vObj != null) { + vObj.setGroupIdList(groups); + vObj.setGroupNameList(groupNames); + } + } + } + } + private void populateUserAttributes(String userName, VXUser vObj) { if (userName != null && !userName.isEmpty()) { List<String> userRoleList =new ArrayList<String>(); @@ -260,6 +348,57 @@ public class XUserService extends XUserServiceBase<XXUser, VXUser> { } } + private void populateUserListAttributes(Map<String, VXUser> userNameVObjMap) { + List<XXPortalUser> allXPortalUsers = daoManager.getXXPortalUser().findAllXPortalUser(); + List<XXPortalUserRole> allXPortalUserRoles = daoManager.getXXPortalUserRole().getAll(); + if (MapUtils.isNotEmpty(userNameVObjMap) && CollectionUtils.isNotEmpty(allXPortalUsers)) { + Map<String, XXPortalUser> loginIdXXPortalUserMap = new HashMap<>(allXPortalUsers.size()); + Map<Long, List<XXPortalUserRole>> userIdRoleMap = new HashMap<>(); + for (XXPortalUser xPortalUser : allXPortalUsers) { + loginIdXXPortalUserMap.put(xPortalUser.getLoginId(), xPortalUser); + List<XXPortalUserRole> xxPortalUserRoles = new ArrayList<>(); + if (allXPortalUserRoles != null) { + for (XXPortalUserRole xPortalUserRole : allXPortalUserRoles) { + if (Objects.equals(xPortalUserRole.getUserId(), xPortalUser.getId())) { + xxPortalUserRoles.add(xPortalUserRole); + } + } + } + userIdRoleMap.put(xPortalUser.getId(), xxPortalUserRoles); + } + + for (Map.Entry<String, VXUser> userNameVObjEntry : userNameVObjMap.entrySet()) { + String userName = userNameVObjEntry.getKey(); + VXUser vObj = userNameVObjEntry.getValue(); + if (userName != null && !userName.isEmpty()) { + List<String> userRoleList = new ArrayList<>(); + XXPortalUser xXPortalUser = loginIdXXPortalUserMap.get(userName); + if (xXPortalUser != null) { + vObj.setFirstName(xXPortalUser.getFirstName()); + vObj.setLastName(xXPortalUser.getLastName()); + vObj.setPassword(PropertiesUtil.getProperty("ranger.password.hidden")); + String emailAddress = xXPortalUser.getEmailAddress(); + if (emailAddress != null + && stringUtil.validateEmail(emailAddress)) { + vObj.setEmailAddress(xXPortalUser.getEmailAddress()); + } + vObj.setStatus(xXPortalUser.getStatus()); + vObj.setUserSource(xXPortalUser.getUserSource()); + List<XXPortalUserRole> gjUserRoleList = userIdRoleMap.get( + xXPortalUser.getId()); + for (XXPortalUserRole gjUserRole : gjUserRoleList) { + userRoleList.add(gjUserRole.getUserRole()); + } + } + if(userRoleList==null || userRoleList.isEmpty()){ + userRoleList.add(RangerConstants.ROLE_USER); + } + vObj.setUserRoleList(userRoleList); + } + } + } + } + public List<XXTrxLog> getTransactionLog(VXUser vResource, String action) { return getTransactionLog(vResource, null, action); } diff --git a/security-admin/src/main/java/org/apache/ranger/service/XUserServiceBase.java b/security-admin/src/main/java/org/apache/ranger/service/XUserServiceBase.java index 9cdc14e..4fcdda2 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/XUserServiceBase.java +++ b/security-admin/src/main/java/org/apache/ranger/service/XUserServiceBase.java @@ -24,17 +24,27 @@ */ import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Function; + import com.google.gson.Gson; import com.google.gson.GsonBuilder; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.MapUtils; +import org.apache.ranger.common.ContextUtil; import org.apache.ranger.common.SearchCriteria; +import org.apache.ranger.common.UserSessionBase; +import org.apache.ranger.entity.XXPortalUser; import org.apache.ranger.entity.XXUser; import org.apache.ranger.plugin.model.UserInfo; import org.apache.ranger.view.VXUser; import org.apache.ranger.view.VXUserList; +import static java.util.stream.Collectors.toMap; + public abstract class XUserServiceBase<T extends XXUser, V extends VXUser> extends AbstractBaseResourceService<T, V> { public static final String NAME = "XUser"; @@ -66,6 +76,126 @@ public abstract class XUserServiceBase<T extends XXUser, V extends VXUser> return vObj; } + protected List<VXUser> mapEntityToViewBeans(Map<VXUser, XXUser> vxUserXXUserMap) { + List<VXUser> vxUsers = new ArrayList<>(); + if (MapUtils.isNotEmpty(vxUserXXUserMap)) { + for (Map.Entry<VXUser, XXUser> vxUserXXUserEntry : vxUserXXUserMap.entrySet()) { + VXUser vObj = vxUserXXUserEntry.getKey(); + XXUser mObj = vxUserXXUserEntry.getValue(); + vObj.setName(mObj.getName()); + vObj.setIsVisible(mObj.getIsVisible()); + vObj.setDescription(mObj.getDescription()); + vObj.setCredStoreId(mObj.getCredStoreId()); + vObj.setOtherAttributes(mObj.getOtherAttributes()); + vxUsers.add(vObj); + } + } + return vxUsers; + } + + public List<VXUser> populateViewBeans(List<XXUser> resources) { + List<VXUser> viewBeans = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(resources)) { + Map<XXUser, VXUser> resourceViewBeanMap = new HashMap<>(resources.size()); + Map<VXUser, XXUser> viewBeanResourceMap = new HashMap<>(resources.size()); + for (XXUser resource : resources) { + VXUser viewBean = createViewObject(); + viewBean.setCredStoreId(resource.getCredStoreId()); + viewBean.setDescription(resource.getDescription()); + viewBean.setName(resource.getName()); + viewBean.setStatus(resource.getStatus()); + resourceViewBeanMap.put(resource, viewBean); + viewBeanResourceMap.put(viewBean, resource); + viewBeans.add(viewBean); + } + populateViewBeans(resourceViewBeanMap); + mapEntityToViewBeans(viewBeanResourceMap); + } + return viewBeans; + } + + protected void populateViewBeans(Map<XXUser, VXUser> resourceViewBeanMap) { + mapBaseAttributesToViewBeans(resourceViewBeanMap); + } + + private void mapBaseAttributesToViewBeans(Map<XXUser, VXUser> resourceViewBeanMap) { + List<XXPortalUser> allXPortalUsers = daoManager.getXXPortalUser().findAllXPortalUser(); + if (MapUtils.isNotEmpty(resourceViewBeanMap) && CollectionUtils.isNotEmpty(allXPortalUsers)) { + Map<Long, XXPortalUser> idXXPortalUserMap = allXPortalUsers + .stream() + .collect(toMap(XXPortalUser::getId, Function.identity())); + resourceViewBeanMap.forEach((resource, viewBean) -> { + viewBean.setId(resource.getId()); + + // TBD: Need to review this change later + viewBean.setMObj(resource); + viewBean.setCreateDate(resource.getCreateTime()); + viewBean.setUpdateDate(resource.getUpdateTime()); + + Long ownerId = resource.getAddedByUserId(); + UserSessionBase currentUserSession = ContextUtil + .getCurrentUserSession(); + + if (currentUserSession == null) { + return; + } + + if (ownerId != null) { + XXPortalUser tUser = idXXPortalUserMap.get( + resource.getAddedByUserId()); + if (tUser != null) { + if (tUser.getPublicScreenName() != null + && !tUser.getPublicScreenName().trim().isEmpty() + && !"null".equalsIgnoreCase(tUser.getPublicScreenName().trim())) { + viewBean.setOwner(tUser.getPublicScreenName()); + } else { + if (tUser.getFirstName() != null + && !tUser.getFirstName().trim().isEmpty() + && !"null".equalsIgnoreCase(tUser.getFirstName().trim())) { + if (tUser.getLastName() != null + && !tUser.getLastName().trim().isEmpty() + && !"null".equalsIgnoreCase(tUser.getLastName().trim())) { + viewBean.setOwner(tUser.getFirstName() + " " + + tUser.getLastName()); + } else { + viewBean.setOwner(tUser.getFirstName()); + } + } else { + viewBean.setOwner(tUser.getLoginId()); + } + } + } + } + if (resource.getUpdatedByUserId() != null) { + XXPortalUser tUser = idXXPortalUserMap.get( + resource.getUpdatedByUserId()); + if (tUser != null) { + if (tUser.getPublicScreenName() != null + && !tUser.getPublicScreenName().trim().isEmpty() + && !"null".equalsIgnoreCase(tUser.getPublicScreenName().trim())) { + viewBean.setUpdatedBy(tUser.getPublicScreenName()); + } else { + if (tUser.getFirstName() != null + && !tUser.getFirstName().trim().isEmpty() + && !"null".equalsIgnoreCase(tUser.getFirstName().trim())) { + if (tUser.getLastName() != null + && !tUser.getLastName().trim().isEmpty() + && !"null".equalsIgnoreCase(tUser.getLastName().trim())) { + viewBean.setUpdatedBy(tUser.getFirstName() + " " + + tUser.getLastName()); + } else { + viewBean.setUpdatedBy(tUser.getFirstName()); + } + } else { + viewBean.setUpdatedBy(tUser.getLoginId()); + } + } + } + } + }); + } + } + /** * @param searchCriteria * @return diff --git a/security-admin/src/main/java/org/apache/ranger/view/VXUser.java b/security-admin/src/main/java/org/apache/ranger/view/VXUser.java index 96f6468..d6f53fd 100644 --- a/security-admin/src/main/java/org/apache/ranger/view/VXUser.java +++ b/security-admin/src/main/java/org/apache/ranger/view/VXUser.java @@ -25,6 +25,7 @@ */ import java.util.Collection; +import java.util.Objects; import javax.xml.bind.annotation.XmlRootElement; @@ -305,6 +306,24 @@ public class VXUser extends VXDataObject implements java.io.Serializable { this.otherAttributes = otherAttributes; } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + VXUser vxUser = (VXUser) o; + + return Objects.equals(status, vxUser.status) && + Objects.equals(name, vxUser.name) && + Objects.equals(description, vxUser.description) && + Objects.equals(credStoreId, vxUser.credStoreId); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), name, description, credStoreId, status); + } + /** * This return the bean content in string format * @return formatedStr
